Updates for recurring execution

2023-09-04 16:39:07 -07:00 · 2023-09-04 16:39:07 -07:00 · ff9b59c96d
parent 626e99da24
commit ff9b59c96d
6 changed files with 190 additions and 9 deletions
--- a/cypress/jenkins/Dockerfile.ci
+++ b/cypress/jenkins/Dockerfile.ci
@ -0,0 +1,3 @@
+FROM cypress/factory
+
+ENTRYPOINT ["bash", "dashboard/cypress/jenkins/cypress.sh"]
--- a/cypress/jenkins/Jenkinsfile
+++ b/cypress/jenkins/Jenkinsfile
@ -34,8 +34,9 @@ node {
                $class: 'GitSCM',
                branches: [[name: "*/${corralBranch}"]],
                extensions: scm.extensions + [[$class: 'CleanCheckout']],
-                userRemoteConfigs: [[url: corralRepo]]
+                userRemoteConfigs: [[url: 'https://github.com/'+corralRepo]]
              ])
+              sh 'git --no-pager branch'
              def ciFilename = "dashboard-tests.yaml"
              def ciConfigContents = env.CORRAL_PACKAGE_CONFIG
              if (ciConfigContents != null) {
@ -45,6 +46,8 @@ node {
        }
        try {
          stage('Run Tests') {
+              sh 'pwd'
+              sh 'ls -al cypress/jenkins'
              sh 'cypress/jenkins/init.sh'
          }
        } catch (err) {
@ -64,6 +67,9 @@ node {
        try {
          stage('Clean Test Environment') {  
            sh "${WORKSPACE}/bin/corral delete ci"
+            if ("${env.JOB_TYPE}" == "recurring") {
+              sh "${WORKSPACE}/bin/corral delete rancher"
+            }
          }
        } catch(err) {
          echo "Error: " + err
--- a/cypress/jenkins/README.md
+++ b/cypress/jenkins/README.md
@ -0,0 +1,122 @@
+# UI Automation execution on Jenkins
+
+## Infrastructure Configuration
+
+The configuration is handled by the [init.sh](./init.sh) script in this repository.
+
+Basically what it does is a setup of a Linux node at a user-level of the software needed to execute the execution script located in the Dashboard tests [Corral Package](https://github.com/rancherlabs/corral-packages).
+
+It configures a `WORKSPACE` folder in the *PATH* to add the necessary binaries.
+
+In Jenkins executors `WORKSPACE` is a predefined variable to a temporary folder in the *jenkins* user home.
+
+- Golang - To build the corral package images.
+- Corral - For executng the corral package to build the remote Node for the tests
+- yq - For updating the corral package configuration values
+
+If the `JOB_TYPE` is `recurring` that will make use of the `rancher` package to spin up a Rancher server for the dashboards tests to run on it.
+
+## Corral configuration
+
+The initialization script makes use of the `corral config vars` option to set the dashboard tests corral package values.
+
+The goal is to have logic in the script for configuring both a Rancher instance and the Dashboard tests node that runs:
+
+ `Dashboard tests node -> Rancher setup`
+
+## Run locally - needs remote aws provider
+
+It is possible to run this locally or in a remove Linux instance.
+
+There's however required environment variables set for configuring the script, download the binary dependencies and execute corral.
+
+This design is mainly due to how the Jenkins Jobs manage the configuration using environment variables.
+
+Some environment have default values thus making them optional others like the following are required:
+
+- `WORKSPACE`
+  - This is defined in Jenkins but it isn't a system variable on Linux.
+- `AWS_ACCESS_KEY_ID`
+- `AWS_SECRET_ACCESS_KEY`
+- `AWS_REGION`
+- `AWS_ZONE_ID`
+- `AWS_DOMAIN`
+- `AWS_AMI`
+- `AWS_SECURITY_GROUP`
+- `AWS_SUBNET`
+- `AWS_VPC`
+- `AWS_VOLUME_TYPE`
+- `AWS_VOLUME_IOPS`
+- `AWS_SSH_USER`
+- `AWS_INSTANCE_TYPE`
+- `RANCHER_TYPE`
+  - This var defines the logic of how to execute the tests by the corral package.
+- `CYPRESS_TAGS`
+
+*For more variables or variables updates take a look at the [init.sh](./init.sh) script.*
+
+Folder structure:
+
+- `WORKSPACE`
+  - `corral-packages` - cloned repo folder.
+  - `dashboard` - cloned repo folder
+  - `bin` - folder with binaries and set in the `PATH`
+
+From `WORKSPACE`:
+
+`dashboard/cypress/jenkins/init.sh`
+
+The expected and default Dashboard corral package image path is:
+
+`${WORKSPACE}/corral-packages/dist/aws-dashboard-tests-t3a.xlarge`
+
+That is generated by `make` during the `init.sh` execution.
+
+### Use run.sh from the corral package directly
+
+The main `run.sh` script has all the logic to directly run the tests without creating the remote AWS node. However that's not fully tested and some modifications might be required to make it work.
+
+## CORRAL PACKAGE
+
+The Dashboard Tests corral package basically create an `AWS` ephimeral node to execute the UI tests on a `cypress` docker container.
+
+The Docker image used is the latest `factory` from the `cypress-io/cypress-docker-images` repository [folder](https://github.com/cypress-io/cypress-docker-images/tree/master/factory).
+That image can recieve arguments for `yarn`, `node`, `cypress` and browsers. This allows the Jenkins job to set these and run.
+
+A [run.sh](https://github.com/izaac/corral-packages/blob/dashboard_tests_recurring/templates/dashboard-tests/overlay/tmp/run.sh) script is executed on the remote node during the node creation.
+
+The script has logic for three different environment configurations - `rancher_type`:
+
+- Execute the tests on an `existing` Rancher setup.
+- Execute the tests in a Rancher instance running along the tests in the `local` node.
+- Do `recurring` execution that on an `existing` ephimeral Rancher created by the automation for Jenkins.
+
+The corral package takes the configuration from the updated/edited YAML files for the `recurring` type and all the variables set by the `init.sh` script with `corral config vars`.
+
+For non recurring types it just make use of the `corral config vars`, as the YAML files are only used for the ephimeral automation Rancher node.
+
+The workflow in `run.sh` of the different `rancher_type`(s).
+
+### Existing
+
+- Install all node modules needed for cypress reporting on the `dashboard` cloned repo.
+- Utilize an existing `rancher_host` username and password.
+- With the username and password, execute a Docker instance with the UI tests
+- Generate jUnit and html reports.
+
+### Recurring
+
+- Install all node modules needed for cypress reporting on the `dashboard` cloned repo.
+- Grab the information of the ephemeral Rancher automation instance from `corral config vars`
+- With that information like the Rancher username and password, execute a Docker instance with the UI tests
+- Generate jUnit and html reports.
+
+### Local
+
+Similar to Drone without the remote reporting and this runs Rancher Docker container and the cypress container.
+This is less used in Jenkins and might be a good option for local testing.
+
+- Install all node modules needed for cypress reporting on the `dashboard` cloned repo.
+- Build UI assets and setup a Docker Rancher instance with them. (Similar to Drone).
+- Attach a second Docker container to the Rancher instance network and execute the UI tests in it.
+- Generate jUnit and html reports.
--- a/cypress/jenkins/cypress.sh
+++ b/cypress/jenkins/cypress.sh
@ -7,7 +7,7 @@ pwd
 ls -al
 cd "dashboard"

-yarn install
+node -v
 yarn add --dev -W mocha mochawesome mochawesome-merge \
  mochawesome-report-generator cypress-multi-reporters \
  mocha-junit-reporter 
--- a/cypress/jenkins/init.sh
+++ b/cypress/jenkins/init.sh
@ -30,9 +30,14 @@ DASHBOARD_REPO="${DASHBOARD_REPO:-rancher/dashboard.git}"
 DASHBOARD_BRANCH="${DASHBOARD_BRANCH:-master}"
 GITHUB_URL="https://github.com/"
 RANCHER_TYPE="${RANCHER_TYPE:-local}"
-NODEJS_VERSION="${NODEJS_VERSION:-v14.19.1}"
+NODEJS_VERSION="${NODEJS_VERSION:-14.19.1}"
+CYPRESS_VERSION="${CYPRESS_VERSION:-13.2.0}"
+YARN_VERSION="${YARN_VERSION:-1.22.19}"
+YQ_BIN="mikefarah/yq/releases/latest/download/yq_linux_amd64"

 mkdir -p "${WORKSPACE}/bin"
+wget "${GITHUB_URL}${YQ_BIN}" -O "${WORKSPACE}/bin/yq"
+chmod +x "${WORKSPACE}/bin/yq"

 if [ -f "${CORRAL}" ]; then rm "${CORRAL}"; fi
 curl -L -o "${CORRAL}" "${CORRAL_DOWNLOAD_BIN}"
@ -43,16 +48,19 @@ tar -C "${WORKSPACE}" -xzf "${GO_PKG_FILENAME}"

 ls -al "${WORKSPACE}"
 export PATH=$PATH:"${WORKSPACE}/go/bin:${WORKSPACE}/bin"
+echo $PATH

 go version

+
 if [[ ! -d "${WORKSPACE}/.ssh" ]]; then mkdir -p "${WORKSPACE}/.ssh"; fi
 export PRIV_KEY="${WORKSPACE}/.ssh/jenkins_ecdsa"
 if [ -f "${PRIV_KEY}" ]; then rm "${PRIV_KEY}"; fi
 ssh-keygen -t ecdsa -b 521 -N "" -f "${PRIV_KEY}"
+ls -al "${WORKSPACE}/.ssh/"

-corral config --public_key "${WORKSPACE}/.ssh/jenkins_ecdsa.pub" --user_id jenkins
-corral config vars set corral_user_public_key "$(cat ${WORKSPACE}/.ssh/jenkins_ecdsa.pub)"
+corral config --public_key "${PRIV_KEY}.pub" --user_id jenkins
+corral config vars set corral_user_public_key "$(cat ${PRIV_KEY}.pub)"
 corral config vars set corral_user_id jenkins
 corral config vars set aws_ssh_user ${AWS_SSH_USER}
 corral config vars set aws_access_key ${AWS_ACCESS_KEY_ID}
@ -62,13 +70,52 @@ corral config vars set aws_region ${AWS_REGION}
 corral config vars set aws_security_group ${AWS_SECURITY_GROUP}
 corral config vars set aws_subnet ${AWS_SUBNET}
 corral config vars set aws_vpc ${AWS_VPC}
+corral config vars set aws_volume_size ${AWS_VOLUME_SIZE}
+corral config vars set aws_volume_type ${AWS_VOLUME_TYPE}
 corral config vars set volume_type ${AWS_VOLUME_TYPE}
 corral config vars set volume_iops ${AWS_VOLUME_IOPS}
+
+if [[ "${JOB_TYPE}" == "recurring" ]]; then 
+  RANCHER_TYPE="recurring"
+  cd "${WORKSPACE}/corral-packages"
+  yq -i e ".variables.rancher_version += [\"${RANCHER_VERSION}\"] | .variables.rancher_version style=\"literal\"" packages/aws/rancher.yaml
+  yq -i e ".variables.kubernetes_version += [\"${RKE2_KUBERNETES_VERSION}\"] | .variables.kubernetes_version style=\"literal\"" packages/aws/rancher.yaml
+  yq -i e ".variables.cert_manager_version += [\"${CERT_MANAGER_VERSION}\"] | .variables.kubernetes_version style=\"literal\"" packages/aws/rancher.yaml
+
+  cat packages/aws/rancher.yaml
+  ls -al packages/aws/
+  cat packages/aws/dashboard-tests.yaml
+
+  prefix_random=$(cat /dev/urandom | env LC_ALL=C tr -dc 'a-z0-9' | fold -w 8 | head -n 1)
+
+  corral config vars set bootstrap_password ${BOOTSTRAP_PASSWORD:-password}
+  corral config vars set aws_route53_zone ${AWS_ROUTE53_ZONE}
+  if [[ -n "${RANCHER_IMAGE_TAG}" ]]; then
+    corral config vars set rancher_image_tag ${RANCHER_IMAGE_TAG}
+  fi
+  corral config vars set server_count ${SERVER_COUNT:-3}
+  corral config vars set agent_count ${AGENT_COUNT:-0}
+  corral config vars set instance_type ${AWS_INSTANCE_TYPE}
+  corral config vars set aws_hostname_prefix "jenkins-${prefix_random}"
+  corral config vars delete rancher_host
+  RANCHER_HOST="jenkins-${prefix_random}.${AWS_ROUTE53_ZONE}"
+
+  RKE2_KUBERNETES_VERSION="${RKE2_KUBERNETES_VERSION//+/-}"
+  make init
+  make build
+  ls -al dist
+  echo "Corral Package string: ${RKE2_KUBERNETES_VERSION}-${RANCHER_VERSION//v}-${CERT_MANAGER_VERSION}"
+  corral create --skip-cleanup --recreate --debug rancher \
+    "dist/aws-rke2-rancher-calico-${RKE2_KUBERNETES_VERSION}-${RANCHER_VERSION//v}-${CERT_MANAGER_VERSION}"
+fi
+
 corral config vars set rancher_type ${RANCHER_TYPE}
 corral config vars set nodejs_version ${NODEJS_VERSION}
 corral config vars set dashboard_repo ${DASHBOARD_REPO}
 corral config vars set dashboard_branch ${DASHBOARD_BRANCH}
 corral config vars set cypress_tags ${CYPRESS_TAGS}
+corral config vars set cypress_version ${CYPRESS_VERSION}
+corral config vars set yarn_version ${YARN_VERSION}

 if [[ -n "${RANCHER_USERNAME}" ]]; then
   corral config vars set rancher_username ${RANCHER_USERNAME}
@ -82,8 +129,8 @@ if [[ -n "${RANCHER_HOST}" ]]; then
   corral config vars set rancher_host ${RANCHER_HOST}
 fi

-if [[ -n "${CYPRESS_DOCKER_BRANCH}" ]]; then
-   corral config vars set cypress_docker_branch ${CYPRESS_DOCKER_BRANCH}
+if [[ -n "${CHROME_VERSION}" ]]; then
+   corral config vars set chrome_version ${CHROME_VERSION}
 fi

 cd "${WORKSPACE}/corral-packages"
@ -91,8 +138,11 @@ make init
 make build
 echo "${PWD}"
 chmod -R 766 "${WORKSPACE}/corral-packages"
+corral config vars set node_count 1
+corral config vars set bastion_ip ""
+corral config vars delete instance_type
 corral create --skip-cleanup --recreate --debug ci dist/aws-dashboard-tests-t3a.xlarge
 corral config vars -o yaml
-NODE_EXTERNAL_IP="$(corral vars ci single_ip)"
+NODE_EXTERNAL_IP="$(corral vars ci first_node_ip)"
 cd ${WORKSPACE}
 echo "${PWD}"
--- a/cypress/jenkins/scpget.sh
+++ b/cypress/jenkins/scpget.sh
@ -5,7 +5,7 @@ set -x
 export PATH=$PATH:"${WORKSPACE}/bin"

 PRIV_KEY="${WORKSPACE}/.ssh/jenkins_ecdsa"
-NODE_EXTERNAL_IP="$(corral vars ci single_ip)"
+NODE_EXTERNAL_IP="$(corral vars ci first_node_ip)"

 scp -r -i ${PRIV_KEY} -o StrictHostKeyChecking=no \
  -o UserKnownHostsFile=/dev/null "${AWS_SSH_USER}@${NODE_EXTERNAL_IP}:$1" .