Merge pull request #10847 from justinsb/pass_provider_with_kubetest2

kubetest2: Infer the provider and zones from the kops cluster

tests/e2e/go.mod

@@ -8,7 +8,61 @@ require (
 	github.com/octago/sflags v0.2.0
 	github.com/spf13/pflag v1.0.5
 	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/apimachinery v0.21.0
 	k8s.io/klog/v2 v2.8.0
+	k8s.io/kops v0.0.0-00010101000000-000000000000
 	sigs.k8s.io/boskos v0.0.0-20200710214748-f5935686c7fc
 	sigs.k8s.io/kubetest2 v0.0.0-20210309183806-9230b4e73d8d
 )
+
+replace k8s.io/kops => ../../.
+
+// These should match the go.mod from k8s.io/kops
+
+replace k8s.io/api => k8s.io/api v0.21.0
+
+replace k8s.io/apimachinery => k8s.io/apimachinery v0.21.0
+
+replace k8s.io/client-go => k8s.io/client-go v0.21.0
+
+replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.21.0
+
+replace k8s.io/controller-manager => k8s.io/controller-manager v0.21.0
+
+replace k8s.io/kubectl => k8s.io/kubectl v0.21.0
+
+replace k8s.io/apiserver => k8s.io/apiserver v0.21.0
+
+replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.0
+
+replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.21.0
+
+replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.21.0
+
+replace k8s.io/cri-api => k8s.io/cri-api v0.21.0
+
+replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.21.0
+
+replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.21.0
+
+replace k8s.io/component-base => k8s.io/component-base v0.21.0
+
+replace k8s.io/component-helpers => k8s.io/component-helpers v0.21.0
+
+replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.21.0
+
+replace k8s.io/metrics => k8s.io/metrics v0.21.0
+
+replace k8s.io/mount-utils => k8s.io/mount-utils v0.21.0
+
+replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.21.0
+
+replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.21.0
+
+replace k8s.io/kubelet => k8s.io/kubelet v0.21.0
+
+replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.21.0
+
+replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.21.0
+
+replace k8s.io/code-generator => k8s.io/code-generator v0.21.0

tests/e2e/kubetest2-kops/deployer/build.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"strings"
 
+	"k8s.io/kops/tests/e2e/pkg/util"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
 
@@ -38,7 +39,8 @@ func (d *deployer) Build() error {
 	if err := d.BuildOptions.Build(); err != nil {
 		return err
 	}
-	return nil
+	// Copy the kops binary into the test's RunDir to be included in the tester's PATH
+	return util.Copy(d.KopsBinaryPath, path.Join(d.commonOptions.RunDir(), "kops"))
 }
 
 func (d *deployer) verifyBuildFlags() error {

tests/e2e/kubetest2-kops/deployer/common.go

@@ -57,13 +57,14 @@ func (d *deployer) initialize() error {
 		}
 	}
 	if d.KopsVersionMarker != "" {
-		binaryPath, baseURL, err := kops.DownloadKops(d.KopsVersionMarker, d.KopsBinaryPath)
+		d.KopsBinaryPath = path.Join(d.commonOptions.RunDir(), "kops")
+		baseURL, err := kops.DownloadKops(d.KopsVersionMarker, d.KopsBinaryPath)
 		if err != nil {
 			return fmt.Errorf("init failed to download kops from url: %v", err)
 		}
-		d.KopsBinaryPath = binaryPath
 		d.KopsBaseURL = baseURL
 	}
+
 	switch d.CloudProvider {
 	case "aws":
 		// These environment variables are defined by the "preset-aws-ssh" prow preset
@@ -105,7 +106,7 @@ func (d *deployer) initialize() error {
 				d.SSHPrivateKeyPath = privateKey
 				d.SSHPublicKeyPath = publicKey
 			}
-			d.createBucket = os.Getenv("KOPS_STATE_STORE") == ""
+			d.createBucket = true
 		}
 	}
 	if d.SSHUser == "" {
@@ -118,6 +119,14 @@ func (d *deployer) initialize() error {
 		}
 		d.terraform = t
 	}
+	if d.commonOptions.ShouldTest() {
+		for _, envvar := range d.env() {
+			// Set all of the env vars we use for kops in the current process
+			// so that the tester inherits them when shelling out to kops
+			i := strings.Index(envvar, "=")
+			os.Setenv(envvar[0:i], envvar[i+1:])
+		}
+	}
 	return nil
 }
 
@@ -133,11 +142,7 @@ func (d *deployer) verifyKopsFlags() error {
 	}
 
 	if d.KopsBinaryPath == "" && d.KopsVersionMarker == "" {
-		if ws := os.Getenv("WORKSPACE"); ws != "" {
+		return errors.New("missing required --kops-binary-path when --kops-version-marker is not used")
-			d.KopsBinaryPath = path.Join(ws, "kops")
-		} else {
-			return errors.New("missing required --kops-binary-path when --kops-version-marker is not used")
-		}
 	}
 
 	switch d.CloudProvider {

tests/e2e/pkg/kops/download.go

@@ -27,39 +27,29 @@ import (
 )
 
 // DownloadKops will download the kops binary from the version marker URL
-// Returning the path to the local kops binary and the URL to use for KOPS_BASE_URL
+// Returning the URL to use for KOPS_BASE_URL
 // Example markerURL: https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt
-func DownloadKops(markerURL, downloadPath string) (string, string, error) {
+func DownloadKops(markerURL, downloadPath string) (string, error) {
 	var b bytes.Buffer
 	if err := util.HTTPGETWithHeaders(markerURL, nil, &b); err != nil {
-		return "", "", err
+		return "", err
 	}
 	kopsBaseURL := strings.TrimSpace(b.String())
 
-	var kopsFile *os.File
+	kopsFile, err := os.Create(downloadPath)
-	if downloadPath == "" {
+	if err != nil {
-		tmp, err := os.CreateTemp("", "kops")
+		return "", err
-		if err != nil {
-			return "", "", err
-		}
-		kopsFile = tmp
-	} else {
-		tmp, err := os.Create(downloadPath)
-		if err != nil {
-			return "", "", err
-		}
-		kopsFile = tmp
 	}
 
 	kopsURL := fmt.Sprintf("%v/%v/%v/kops", kopsBaseURL, runtime.GOOS, runtime.GOARCH)
 	if err := util.HTTPGETWithHeaders(kopsURL, nil, kopsFile); err != nil {
-		return "", "", err
+		return "", err
 	}
 	if err := kopsFile.Close(); err != nil {
-		return "", "", err
+		return "", err
 	}
 	if err := os.Chmod(kopsFile.Name(), 0755); err != nil {
-		return "", "", err
+		return "", err
 	}
-	return kopsFile.Name(), kopsBaseURL, nil
+	return kopsBaseURL, nil
 }

tests/e2e/pkg/kops/state.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kops
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"k8s.io/klog/v2"
+	api "k8s.io/kops/pkg/apis/kops/v1alpha2"
+)
+
+// GetCluster will retrieve the specified Cluster from the state store.
+func GetCluster(clusterName string) (*api.Cluster, error) {
+	args := []string{
+		"kops", "get", "cluster", clusterName, "-ojson",
+	}
+	c := exec.Command(args[0], args[1:]...)
+	var stdout bytes.Buffer
+	c.Stdout = &stdout
+	var stderr bytes.Buffer
+	c.Stderr = &stderr
+	if err := c.Run(); err != nil {
+		klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
+		return nil, fmt.Errorf("error querying cluster from %s: %w", strings.Join(args, " "), err)
+	}
+
+	cluster := &api.Cluster{}
+	if err := json.Unmarshal(stdout.Bytes(), cluster); err != nil {
+		return nil, fmt.Errorf("error parsing cluster json: %w", err)
+	}
+	return cluster, nil
+}
+
+// GetInstanceGroups will retrieve the instance groups for the specified Cluster from the state store.
+func GetInstanceGroups(clusterName string) ([]*api.InstanceGroup, error) {
+	args := []string{
+		"kops", "get", "instancegroups", "--name", clusterName, "-ojson",
+	}
+	c := exec.Command(args[0], args[1:]...)
+	var stdout bytes.Buffer
+	c.Stdout = &stdout
+	var stderr bytes.Buffer
+	c.Stderr = &stderr
+	if err := c.Run(); err != nil {
+		klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
+		return nil, fmt.Errorf("error querying instance groups from %s: %w", strings.Join(args, " "), err)
+	}
+
+	var igs []*api.InstanceGroup
+	if err := json.Unmarshal(stdout.Bytes(), &igs); err != nil {
+		return nil, fmt.Errorf("error parsing instance groups json: %w", err)
+	}
+	return igs, nil
+}

tests/e2e/pkg/tester/tester.go

@@ -25,14 +25,20 @@ import (
 	"strings"
 
 	"github.com/octago/sflags/gen/gpflag"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
-
 	"sigs.k8s.io/kubetest2/pkg/testers/ginkgo"
+
+	api "k8s.io/kops/pkg/apis/kops/v1alpha2"
+	"k8s.io/kops/tests/e2e/pkg/kops"
 )
 
 // Tester wraps kubetest2's ginkgo tester with additional functionality
 type Tester struct {
 	*ginkgo.Tester
+
+	kopsCluster        *api.Cluster
+	kopsInstanceGroups []*api.InstanceGroup
 }
 
 func (t *Tester) pretestSetup() error {
@@ -47,11 +53,10 @@ func (t *Tester) pretestSetup() error {
 	return os.Setenv("PATH", newPath)
 }
 
-// The --host argument was required in the kubernetes e2e tests, until https://github.com/kubernetes/kubernetes/pull/87030
+// parseKubeconfig will get the current kubeconfig, and extract the specified field by jsonpath.
-// We can likely drop this when we drop support / testing for k8s 1.17
+func parseKubeconfig(jsonPath string) (string, error) {
-func (t *Tester) addHostArgument() error {
 	args := []string{
-		"kubectl", "config", "view", "--minify", "-o", "jsonpath='{.clusters[0].cluster.server}'",
+		"kubectl", "config", "view", "--minify", "-o", "jsonpath={" + jsonPath + "}",
 	}
 	c := exec.Command(args[0], args[1:]...)
 	var stdout bytes.Buffer
@@ -60,19 +65,241 @@ func (t *Tester) addHostArgument() error {
 	c.Stderr = &stderr
 	if err := c.Run(); err != nil {
 		klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
-		return fmt.Errorf("error querying current config from kubectl: %w", err)
+		return "", fmt.Errorf("error querying current config from kubectl: %w", err)
 	}
 
-	server := strings.TrimSpace(stdout.String())
+	s := strings.TrimSpace(stdout.String())
-	if server == "" {
+	if s == "" {
-		return fmt.Errorf("kubeconfig did not contain server")
+		return "", fmt.Errorf("kubeconfig did not contain " + jsonPath)
 	}
+	return s, nil
+}
 
+// The --host flag was required in the kubernetes e2e tests, until https://github.com/kubernetes/kubernetes/pull/87030
+// We can likely drop this when we drop support / testing for k8s 1.17
+func (t *Tester) addHostFlag() error {
+	server, err := parseKubeconfig(".clusters[0].cluster.server")
+	if err != nil {
+		return err
+	}
 	klog.Infof("Adding --host=%s", server)
 	t.TestArgs += " --host=" + server
 	return nil
 }
 
+// hasFlag detects if the specified flag has been passed in the args
+func hasFlag(args string, flag string) bool {
+	for _, arg := range strings.Split(args, " ") {
+		if !strings.HasPrefix(arg, "-") {
+			continue
+		}
+
+		arg = strings.TrimLeft(arg, "-")
+		if arg == flag || strings.HasPrefix(arg, flag+"=") {
+			return true
+		}
+	}
+	return false
+}
+
+func (t *Tester) getKopsCluster() (*api.Cluster, error) {
+	if t.kopsCluster != nil {
+		return t.kopsCluster, nil
+	}
+
+	currentContext, err := parseKubeconfig(".current-context")
+	if err != nil {
+		return nil, err
+	}
+
+	kopsClusterName := currentContext
+
+	cluster, err := kops.GetCluster(kopsClusterName)
+	if err != nil {
+		return nil, err
+	}
+	t.kopsCluster = cluster
+
+	return cluster, nil
+
+}
+
+func (t *Tester) getKopsInstanceGroups() ([]*api.InstanceGroup, error) {
+	if t.kopsInstanceGroups != nil {
+		return t.kopsInstanceGroups, nil
+	}
+
+	cluster, err := t.getKopsCluster()
+	if err != nil {
+		return nil, err
+	}
+
+	igs, err := kops.GetInstanceGroups(cluster.Name)
+	if err != nil {
+		return nil, err
+	}
+	t.kopsInstanceGroups = igs
+
+	return igs, nil
+
+}
+func (t *Tester) addProviderFlag() error {
+	if hasFlag(t.TestArgs, "provider") {
+		return nil
+	}
+
+	cluster, err := t.getKopsCluster()
+	if err != nil {
+		return err
+	}
+
+	provider := ""
+	switch cluster.Spec.CloudProvider {
+	case "aws", "gce":
+		provider = cluster.Spec.CloudProvider
+	default:
+		return fmt.Errorf("unhandled cluster.spec.cloudProvider %q for determining ginkgo Provider", cluster.Spec.CloudProvider)
+	}
+
+	klog.Infof("Setting --provider=%s", provider)
+	t.TestArgs += " --provider=" + provider
+	return nil
+}
+
+func (t *Tester) addZoneFlag() error {
+	// gce-zone is indeed used for AWS as well!
+	if hasFlag(t.TestArgs, "gce-zone") {
+		return nil
+	}
+
+	zoneNames, err := t.getZones()
+	if err != nil {
+		return err
+	}
+
+	// gce-zone only expects one zone, we just pass the first one
+	zone := zoneNames[0]
+	klog.Infof("Setting --gce-zone=%s", zone)
+	t.TestArgs += " --gce-zone=" + zone
+
+	// TODO: Pass the new gce-zones flag for 1.21 with all zones?
+
+	return nil
+}
+
+func (t *Tester) addMultiZoneFlag() error {
+	if hasFlag(t.TestArgs, "gce-multizone") {
+		return nil
+	}
+
+	zoneNames, err := t.getZones()
+	if err != nil {
+		return err
+	}
+
+	klog.Infof("Setting --gce-multizone=%t", len(zoneNames) > 1)
+	t.TestArgs += fmt.Sprintf(" --gce-multizone=%t", len(zoneNames) > 1)
+
+	return nil
+}
+
+func (t *Tester) addRegionFlag() error {
+	// gce-zone is used for other cloud providers as well
+	if hasFlag(t.TestArgs, "gce-region") {
+		return nil
+	}
+
+	cluster, err := t.getKopsCluster()
+	if err != nil {
+		return err
+	}
+
+	// We don't explicitly set the provider's region in the spec so we need to extract it from vairous fields
+	var region string
+	switch cluster.Spec.CloudProvider {
+	case "aws":
+		zone := cluster.Spec.Subnets[0].Zone
+		region = zone[:len(zone)-1]
+	case "gce":
+		region = cluster.Spec.Subnets[0].Region
+	default:
+		return fmt.Errorf("unhandled region detection for cloud provider: %v", cluster.Spec.CloudProvider)
+	}
+
+	klog.Infof("Setting --gce-region=%s", region)
+	t.TestArgs += " --gce-region=" + region
+	return nil
+}
+
+func (t *Tester) addClusterTagFlag() error {
+	if hasFlag(t.TestArgs, "cluster-tag") {
+		return nil
+	}
+
+	cluster, err := t.getKopsCluster()
+	if err != nil {
+		return err
+	}
+
+	clusterName := cluster.ObjectMeta.Name
+	klog.Infof("Setting --cluster-tag=%s", clusterName)
+	t.TestArgs += " --cluster-tag=" + clusterName
+
+	return nil
+}
+
+func (t *Tester) addProjectFlag() error {
+	if hasFlag(t.TestArgs, "gce-project") {
+		return nil
+	}
+
+	cluster, err := t.getKopsCluster()
+	if err != nil {
+		return err
+	}
+
+	projectID := cluster.Spec.Project
+	if projectID == "" {
+		return nil
+	}
+	klog.Infof("Setting --gce-project=%s", projectID)
+	t.TestArgs += " --gce-project=" + projectID
+
+	return nil
+}
+
+func (t *Tester) getZones() ([]string, error) {
+	cluster, err := t.getKopsCluster()
+	if err != nil {
+		return nil, err
+	}
+
+	igs, err := t.getKopsInstanceGroups()
+	if err != nil {
+		return nil, err
+	}
+
+	zones := sets.NewString()
+	// Gather zones on AWS
+	for _, subnet := range cluster.Spec.Subnets {
+		if subnet.Zone != "" {
+			zones.Insert(subnet.Zone)
+		}
+	}
+	// Gather zones on GCE
+	for _, ig := range igs {
+		for _, zone := range ig.Spec.Zones {
+			zones.Insert(zone)
+		}
+	}
+	zoneNames := zones.List()
+
+	if len(zoneNames) == 0 {
+		return nil, fmt.Errorf("no zones found in instance groups")
+	}
+	return zoneNames, nil
+}
+
 func (t *Tester) execute() error {
 	fs, err := gpflag.Parse(t)
 	if err != nil {
@@ -94,7 +321,31 @@ func (t *Tester) execute() error {
 		return err
 	}
 
-	if err := t.addHostArgument(); err != nil {
+	if err := t.addHostFlag(); err != nil {
+		return err
+	}
+
+	if err := t.addProviderFlag(); err != nil {
+		return err
+	}
+
+	if err := t.addZoneFlag(); err != nil {
+		return err
+	}
+
+	if err := t.addClusterTagFlag(); err != nil {
+		return err
+	}
+
+	if err := t.addRegionFlag(); err != nil {
+		return err
+	}
+
+	if err := t.addMultiZoneFlag(); err != nil {
+		return err
+	}
+
+	if err := t.addProjectFlag(); err != nil {
 		return err
 	}
 
@@ -102,9 +353,9 @@ func (t *Tester) execute() error {
 }
 
 func NewDefaultTester() *Tester {
-	return &Tester{
+	t := &Tester{}
-		ginkgo.NewDefaultTester(),
+	t.Tester = ginkgo.NewDefaultTester()
-	}
+	return t
 }
 
 func Main() {

tests/e2e/pkg/tester/tester_test.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tester
+
+import (
+	"testing"
+
+	"github.com/octago/sflags/gen/gpflag"
+)
+
+func TestFlagParsing(t *testing.T) {
+	tester := &Tester{}
+
+	fs, err := gpflag.Parse(tester)
+	if err != nil {
+		t.Fatalf("gpflag.Parse(tester) failed: %v", err)
+	}
+
+	args := []string{"--parallel", "25"}
+	if err := fs.Parse(args); err != nil {
+		t.Fatalf("fs.Parse(args) failed: %v", err)
+	}
+
+	if tester.Parallel != 25 {
+		t.Errorf("unexpected value for Parallel; got %d, want %d", tester.Parallel, 25)
+	}
+
+}
+
+func TestHasFlag(t *testing.T) {
+	grid := []struct {
+		Args     string
+		Flag     string
+		Expected bool
+	}{
+		{
+			Args:     "--provider aws",
+			Flag:     "provider",
+			Expected: true,
+		},
+		{
+			Args:     "-provider aws",
+			Flag:     "provider",
+			Expected: true,
+		},
+		{
+			Args:     "provider aws",
+			Flag:     "provider",
+			Expected: false,
+		},
+		{
+			Args:     "-provider=aws",
+			Flag:     "provider",
+			Expected: true,
+		},
+		{
+			Args:     "--provider=aws",
+			Flag:     "provider",
+			Expected: true,
+		},
+		{
+			Args:     "--foo=bar --provider aws",
+			Flag:     "provider",
+			Expected: true,
+		},
+		{
+			Args:     "--foo=bar",
+			Flag:     "provider",
+			Expected: false,
+		},
+	}
+
+	for _, g := range grid {
+		t.Run(g.Args, func(t *testing.T) {
+			got := hasFlag(g.Args, g.Flag)
+			if got != g.Expected {
+				t.Errorf("hasFlags(%q, %q) got %v, want %v", g.Args, g.Flag, got, g.Expected)
+			}
+		})
+	}
+}

tests/e2e/pkg/util/copy.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"strings"
+
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/kubetest2/pkg/exec"
+)
+
+func Copy(src, dest string) error {
+	cpArgs := []string{
+		"cp", src, dest,
+	}
+	klog.Info(strings.Join(cpArgs, " "))
+	cmd := exec.Command(cpArgs[0], cpArgs[1:]...)
+	exec.InheritOutput(cmd)
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}