From 0bbaaa37429391fca57ea18e5bb01b04227d3ad1 Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Thu, 11 Mar 2021 16:47:29 -0600 Subject: [PATCH] Kubetest2 - use same kops binary for all commands in upgrade scenario This ensures that the same binary is used for kubetest2-kops commands as well as the kops commands invoked directly in the scenario script. Periodic jobs will create a temp file that will be used to save the kops binary from the provided version marker. non-periodic jobs (local invocation) will use the bazel build binary, preserving original behavior but using this same binary for kops commands rather than relying on PATH. --- tests/e2e/kubetest2-kops/deployer/common.go | 2 +- tests/e2e/pkg/kops/download.go | 26 ++++++++++++++------- tests/e2e/scenarios/upgrade/run-test | 23 ++++++++++-------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go index f5bc1a2686..6cc0277c1e 100644 --- a/tests/e2e/kubetest2-kops/deployer/common.go +++ b/tests/e2e/kubetest2-kops/deployer/common.go @@ -54,7 +54,7 @@ func (d *deployer) initialize() error { } } if d.KopsVersionMarker != "" { - binaryPath, baseURL, err := kops.DownloadKops(d.KopsVersionMarker) + binaryPath, baseURL, err := kops.DownloadKops(d.KopsVersionMarker, d.KopsBinaryPath) if err != nil { return fmt.Errorf("init failed to download kops from url: %v", err) } diff --git a/tests/e2e/pkg/kops/download.go b/tests/e2e/pkg/kops/download.go index 250b9b75b5..548ed1d6a6 100644 --- a/tests/e2e/pkg/kops/download.go +++ b/tests/e2e/pkg/kops/download.go @@ -29,27 +29,37 @@ import ( // DownloadKops will download the kops binary from the version marker URL // Returning the path to the local kops binary and the URL to use for KOPS_BASE_URL // Example markerURL: https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt -func DownloadKops(markerURL string) (string, string, error) { +func DownloadKops(markerURL, downloadPath string) (string, string, error) { var b bytes.Buffer if err := util.HTTPGETWithHeaders(markerURL, nil, &b); err != nil { return "", "", err } kopsBaseURL := b.String() - tmp, err := ioutil.TempFile("", "kops") - if err != nil { - return "", "", err + var kopsFile *os.File + if downloadPath == "" { + tmp, err := ioutil.TempFile("", "kops") + if err != nil { + return "", "", err + } + kopsFile = tmp + } else { + tmp, err := os.Create(downloadPath) + if err != nil { + return "", "", err + } + kopsFile = tmp } kopsURL := fmt.Sprintf("%v/%v/%v/kops", kopsBaseURL, runtime.GOOS, runtime.GOARCH) - if err := util.HTTPGETWithHeaders(kopsURL, nil, tmp); err != nil { + if err := util.HTTPGETWithHeaders(kopsURL, nil, kopsFile); err != nil { return "", "", err } - if err := tmp.Close(); err != nil { + if err := kopsFile.Close(); err != nil { return "", "", err } - if err := os.Chmod(tmp.Name(), 0755); err != nil { + if err := os.Chmod(kopsFile.Name(), 0755); err != nil { return "", "", err } - return tmp.Name(), kopsBaseURL, nil + return kopsFile.Name(), kopsBaseURL, nil } diff --git a/tests/e2e/scenarios/upgrade/run-test b/tests/e2e/scenarios/upgrade/run-test index 0c2cd4733a..3fd9a13ed4 100755 --- a/tests/e2e/scenarios/upgrade/run-test +++ b/tests/e2e/scenarios/upgrade/run-test @@ -35,9 +35,12 @@ go install ./kubetest2-kops go install ./kubetest2-tester-kops if [[ "${JOB_TYPE}" == "periodic" ]]; then + KOPS=$(mktemp).kops KUBETEST2_COMMON_ARGS="${KUBETEST2_COMMON_ARGS} --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt" + KUBETEST2_COMMON_ARGS="${KUBETEST2_COMMON_ARGS} --kops-binary-path=${KOPS}" else - KUBETEST2_COMMON_ARGS="${KUBETEST2_COMMON_ARGS} --kops-binary-path=${REPO_ROOT}/bazel-bin/cmd/kops/linux-amd64/kops" + KOPS=${REPO_ROOT}/bazel-bin/cmd/kops/$(go env GOOS)-$(go env GOARCH)/kops + KUBETEST2_COMMON_ARGS="${KUBETEST2_COMMON_ARGS} --kops-binary-path=${KOPS}" kubetest2 kops ${KUBETEST2_COMMON_ARGS} --build --kops-root=${REPO_ROOT} --stage-location=${STAGE_LOCATION:-} fi @@ -55,15 +58,15 @@ kubetest2 kops ${KUBETEST2_COMMON_ARGS} \ KUBECONFIG=${HOME}/.kube/config TEST_ARGS="--kubeconfig=${KUBECONFIG}" if [[ "${CLOUD_PROVIDER}" == "aws" ]]; then - ZONES=`kops get cluster ${CLUSTER_NAME} -ojson | jq -r .spec.subnets[].zone` + ZONES=`${KOPS} get cluster ${CLUSTER_NAME} -ojson | jq -r .spec.subnets[].zone` CLUSTER_TAG="${CLUSTER_NAME}" TEST_ARGS="${TEST_ARGS} --provider=aws --cluster-tag=${CLUSTER_TAG}" # For historical reasons, the flag name for e2e tests on AWS is --gce-zone TEST_ARGS="${TEST_ARGS} --gce-zone=${ZONES[0]}" fi if [[ "${CLOUD_PROVIDER}" == "gce" ]]; then - ZONES=`kops get ig --name ${CLUSTER_NAME} -ojson | jq -r .[0].spec.zones[]` - GCE_PROJECT=`kops get cluster ${CLUSTER_NAME} -ojson |jq -r .spec.project` + ZONES=`${KOPS} get ig --name ${CLUSTER_NAME} -ojson | jq -r .[0].spec.zones[]` + GCE_PROJECT=`${KOPS} get cluster ${CLUSTER_NAME} -ojson |jq -r .spec.project` TEST_ARGS="${TEST_ARGS} --provider=gce --gce-zone=${ZONES[0]} --gce-project=${GCE_PROJECT}" fi @@ -77,11 +80,11 @@ kubetest2 kops ${KUBETEST2_COMMON_ARGS} \ --test-args="${TEST_ARGS}" -kops set cluster ${CLUSTER_NAME} cluster.spec.kubernetesVersion=v1.19.7 -kops update cluster -kops update cluster --admin --yes +${KOPS} set cluster ${CLUSTER_NAME} cluster.spec.kubernetesVersion=v1.19.7 +${KOPS} update cluster +${KOPS} update cluster --admin --yes -kops rolling-update cluster -kops rolling-update cluster --yes --validation-timeout 30m +${KOPS} rolling-update cluster +${KOPS} rolling-update cluster --yes --validation-timeout 30m -kops validate cluster +${KOPS} validate cluster