Merge pull request #11017 from rifelpet/kubetest2

Kubetest2 - use same kops binary for all commands in upgrade scenario

tests/e2e/kubetest2-kops/deployer/common.go

@@ -54,7 +54,7 @@ func (d *deployer) initialize() error {
 		}
 	}
 	if d.KopsVersionMarker != "" {
-		binaryPath, baseURL, err := kops.DownloadKops(d.KopsVersionMarker)
+		binaryPath, baseURL, err := kops.DownloadKops(d.KopsVersionMarker, d.KopsBinaryPath)
 		if err != nil {
 			return fmt.Errorf("init failed to download kops from url: %v", err)
 		}

tests/e2e/pkg/kops/download.go

@@ -29,27 +29,37 @@ import (
 // DownloadKops will download the kops binary from the version marker URL
 // Returning the path to the local kops binary and the URL to use for KOPS_BASE_URL
 // Example markerURL: https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt
-func DownloadKops(markerURL string) (string, string, error) {
+func DownloadKops(markerURL, downloadPath string) (string, string, error) {
 	var b bytes.Buffer
 	if err := util.HTTPGETWithHeaders(markerURL, nil, &b); err != nil {
 		return "", "", err
 	}
 	kopsBaseURL := b.String()
 
-	tmp, err := ioutil.TempFile("", "kops")
+	var kopsFile *os.File
-	if err != nil {
+	if downloadPath == "" {
-		return "", "", err
+		tmp, err := ioutil.TempFile("", "kops")
+		if err != nil {
+			return "", "", err
+		}
+		kopsFile = tmp
+	} else {
+		tmp, err := os.Create(downloadPath)
+		if err != nil {
+			return "", "", err
+		}
+		kopsFile = tmp
 	}
 
 	kopsURL := fmt.Sprintf("%v/%v/%v/kops", kopsBaseURL, runtime.GOOS, runtime.GOARCH)
-	if err := util.HTTPGETWithHeaders(kopsURL, nil, tmp); err != nil {
+	if err := util.HTTPGETWithHeaders(kopsURL, nil, kopsFile); err != nil {
 		return "", "", err
 	}
-	if err := tmp.Close(); err != nil {
+	if err := kopsFile.Close(); err != nil {
 		return "", "", err
 	}
-	if err := os.Chmod(tmp.Name(), 0755); err != nil {
+	if err := os.Chmod(kopsFile.Name(), 0755); err != nil {
 		return "", "", err
 	}
-	return tmp.Name(), kopsBaseURL, nil
+	return kopsFile.Name(), kopsBaseURL, nil
 }

tests/e2e/scenarios/upgrade/run-test

@@ -35,9 +35,12 @@ go install ./kubetest2-kops
 go install ./kubetest2-tester-kops
 
 if [[ "${JOB_TYPE}" == "periodic" ]]; then
+  KOPS=$(mktemp).kops
   KUBETEST2_COMMON_ARGS="${KUBETEST2_COMMON_ARGS} --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt"
+  KUBETEST2_COMMON_ARGS="${KUBETEST2_COMMON_ARGS} --kops-binary-path=${KOPS}"
 else
-  KUBETEST2_COMMON_ARGS="${KUBETEST2_COMMON_ARGS} --kops-binary-path=${REPO_ROOT}/bazel-bin/cmd/kops/linux-amd64/kops"
+  KOPS=${REPO_ROOT}/bazel-bin/cmd/kops/$(go env GOOS)-$(go env GOARCH)/kops
+  KUBETEST2_COMMON_ARGS="${KUBETEST2_COMMON_ARGS} --kops-binary-path=${KOPS}"
   kubetest2 kops ${KUBETEST2_COMMON_ARGS} --build --kops-root=${REPO_ROOT} --stage-location=${STAGE_LOCATION:-}
 fi
 
@@ -55,15 +58,15 @@ kubetest2 kops ${KUBETEST2_COMMON_ARGS} \
 KUBECONFIG=${HOME}/.kube/config
 TEST_ARGS="--kubeconfig=${KUBECONFIG}"
 if [[ "${CLOUD_PROVIDER}" == "aws" ]]; then
-  ZONES=`kops get cluster ${CLUSTER_NAME} -ojson | jq -r .spec.subnets[].zone`
+  ZONES=`${KOPS} get cluster ${CLUSTER_NAME} -ojson | jq -r .spec.subnets[].zone`
   CLUSTER_TAG="${CLUSTER_NAME}"
   TEST_ARGS="${TEST_ARGS} --provider=aws --cluster-tag=${CLUSTER_TAG}"
   # For historical reasons, the flag name for e2e tests on AWS is --gce-zone
   TEST_ARGS="${TEST_ARGS} --gce-zone=${ZONES[0]}"
 fi
 if [[ "${CLOUD_PROVIDER}" == "gce" ]]; then
-  ZONES=`kops get ig --name ${CLUSTER_NAME} -ojson | jq -r .[0].spec.zones[]`
+  ZONES=`${KOPS} get ig --name ${CLUSTER_NAME} -ojson | jq -r .[0].spec.zones[]`
-  GCE_PROJECT=`kops get cluster ${CLUSTER_NAME} -ojson |jq -r .spec.project`
+  GCE_PROJECT=`${KOPS} get cluster ${CLUSTER_NAME} -ojson |jq -r .spec.project`
   TEST_ARGS="${TEST_ARGS} --provider=gce --gce-zone=${ZONES[0]} --gce-project=${GCE_PROJECT}"
 fi
 
@@ -77,11 +80,11 @@ kubetest2 kops ${KUBETEST2_COMMON_ARGS} \
 		--test-args="${TEST_ARGS}"
 
 
-kops set cluster ${CLUSTER_NAME} cluster.spec.kubernetesVersion=v1.19.7
+${KOPS} set cluster ${CLUSTER_NAME} cluster.spec.kubernetesVersion=v1.19.7
-kops update cluster
+${KOPS} update cluster
-kops update cluster --admin --yes
+${KOPS} update cluster --admin --yes
 
-kops rolling-update cluster
+${KOPS} rolling-update cluster
-kops rolling-update cluster --yes --validation-timeout 30m
+${KOPS} rolling-update cluster --yes --validation-timeout 30m
 
-kops validate cluster
+${KOPS} validate cluster