From 6bdefda4ecc69c44b79ba6643cf4e887e613b616 Mon Sep 17 00:00:00 2001 From: upodroid Date: Fri, 22 Sep 2023 19:32:15 +0100 Subject: [PATCH] fix bugs in e2e testing and add kube feature gates --- tests/e2e/e2e.mk | 1 - tests/e2e/go.mod | 2 +- tests/e2e/kubetest2-kops/deployer/deployer.go | 15 ++++---- tests/e2e/kubetest2-kops/deployer/dumplogs.go | 5 --- tests/e2e/kubetest2-kops/deployer/up.go | 34 +++++++++++++++---- tests/e2e/kubetest2-kops/gce/gcs.go | 2 +- tests/e2e/pkg/util/publicip.go | 21 ++++-------- 7 files changed, 44 insertions(+), 36 deletions(-) diff --git a/tests/e2e/e2e.mk b/tests/e2e/e2e.mk index 1d8018002b..a0746321c0 100644 --- a/tests/e2e/e2e.mk +++ b/tests/e2e/e2e.mk @@ -18,7 +18,6 @@ GOARCH ?= $(shell go env GOARCH) .PHONY: test-e2e-install test-e2e-install: cd $(KOPS_ROOT)/tests/e2e && \ - go install sigs.k8s.io/kubetest2 && \ go install ./kubetest2-tester-kops && \ go install ./kubetest2-kops diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod index cc595fd00d..ede1245a3d 100644 --- a/tests/e2e/go.mod +++ b/tests/e2e/go.mod @@ -17,6 +17,7 @@ require ( k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible k8s.io/klog/v2 v2.100.1 k8s.io/kops v1.24.1 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/boskos v0.0.0-20220704141725-37bd9bb41b86 sigs.k8s.io/kubetest2 v0.0.0-20220801170629-1284e5ada592 sigs.k8s.io/yaml v1.3.0 @@ -139,7 +140,6 @@ require ( k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect k8s.io/release v0.7.1-0.20210204090829-09fb5e3883b8 // indirect k8s.io/test-infra v0.0.0-20210730160938-8ad9b8c53bd8 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect ) diff --git a/tests/e2e/kubetest2-kops/deployer/deployer.go b/tests/e2e/kubetest2-kops/deployer/deployer.go index 54d6ea69d5..ff10ae201f 100644 --- a/tests/e2e/kubetest2-kops/deployer/deployer.go +++ b/tests/e2e/kubetest2-kops/deployer/deployer.go @@ -48,13 +48,14 @@ type deployer struct { KopsBaseURL string `flag:"-"` PublishVersionMarker string `flag:"publish-version-marker" desc:"The GCS path to which the --kops-version-marker is uploaded if the tests pass"` - ClusterName string `flag:"cluster-name" desc:"The FQDN to use for the cluster name"` - CloudProvider string `flag:"cloud-provider" desc:"Which cloud provider to use"` - GCPProject string `flag:"gcp-project" desc:"Which GCP Project to use when --cloud-provider=gce"` - Env []string `flag:"env" desc:"Additional env vars to set for kops commands in NAME=VALUE format"` - CreateArgs string `flag:"create-args" desc:"Extra space-separated arguments passed to 'kops create cluster'"` - KopsBinaryPath string `flag:"kops-binary-path" desc:"The path to kops executable used for testing"` - createBucket bool `flag:"-"` + ClusterName string `flag:"cluster-name" desc:"The FQDN to use for the cluster name"` + CloudProvider string `flag:"cloud-provider" desc:"Which cloud provider to use"` + GCPProject string `flag:"gcp-project" desc:"Which GCP Project to use when --cloud-provider=gce"` + Env []string `flag:"env" desc:"Additional env vars to set for kops commands in NAME=VALUE format"` + CreateArgs string `flag:"create-args" desc:"Extra space-separated arguments passed to 'kops create cluster'"` + KopsBinaryPath string `flag:"kops-binary-path" desc:"The path to kops executable used for testing"` + KubernetesFeatureGates string `flag:"kubernetes-feature-gates" desc:"Feature Gates to enable on Kubernetes components"` + createBucket bool `flag:"-"` // ControlPlaneCount specifies the number of VMs in the control-plane. ControlPlaneCount int `flag:"control-plane-count" desc:"Number of control-plane instances"` diff --git a/tests/e2e/kubetest2-kops/deployer/dumplogs.go b/tests/e2e/kubetest2-kops/deployer/dumplogs.go index e22beea85d..0f8a8ba0bd 100644 --- a/tests/e2e/kubetest2-kops/deployer/dumplogs.go +++ b/tests/e2e/kubetest2-kops/deployer/dumplogs.go @@ -281,8 +281,3 @@ func findControlPlaneIPUser(dump resources.Dump) (string, string, bool) { klog.Warning("ControlPlane instance not found from kops toolbox dump") return "", "", false } - -func runWithOutput(cmd exec.Cmd) error { - exec.InheritOutput(cmd) - return cmd.Run() -} diff --git a/tests/e2e/kubetest2-kops/deployer/up.go b/tests/e2e/kubetest2-kops/deployer/up.go index 2accf227ae..c5f47e95e1 100644 --- a/tests/e2e/kubetest2-kops/deployer/up.go +++ b/tests/e2e/kubetest2-kops/deployer/up.go @@ -32,6 +32,7 @@ import ( "k8s.io/kops/tests/e2e/pkg/kops" "k8s.io/kops/tests/e2e/pkg/util" "k8s.io/kops/tests/e2e/pkg/version" + "k8s.io/utils/strings/slices" "sigs.k8s.io/kubetest2/pkg/exec" ) @@ -152,21 +153,31 @@ func (d *deployer) createCluster(zones []string, adminAccess string, yes bool) e args = appendIfUnset(args, "--master-volume-size", "48") args = appendIfUnset(args, "--node-count", "4") args = appendIfUnset(args, "--node-volume-size", "48") - args = appendIfUnset(args, "--set", adminAccess) args = appendIfUnset(args, "--zones", strings.Join(zones, ",")) switch d.CloudProvider { case "aws": if isArm { args = appendIfUnset(args, "--master-size", "c6g.large") + args = appendIfUnset(args, "--node-size", "c6g.large") } else { args = appendIfUnset(args, "--master-size", "c5.large") } case "gce": - args = appendIfUnset(args, "--master-size", "e2-standard-2") + if isArm { + args = appendIfUnset(args, "--master-size", "t2a-standard-2") + args = appendIfUnset(args, "--node-size", "t2a-standard-2") + } else { + args = appendIfUnset(args, "--master-size", "e2-standard-2") + args = appendIfUnset(args, "--node-size", "e2-standard-2") + } if d.GCPProject != "" { args = appendIfUnset(args, "--project", d.GCPProject) } + // set some sane default e2e testing behaviour on gce + args = appendIfUnset(args, "--gce-service-account", "default") + args = appendIfUnset(args, "--networking", "kubenet") + // We used to set the --vpc flag to split clusters into different networks, this is now the default. // args = appendIfUnset(args, "--vpc", strings.Split(d.ClusterName, ".")[0]) case "digitalocean": @@ -178,7 +189,10 @@ func (d *deployer) createCluster(zones []string, adminAccess string, yes bool) e args = append(args, "--target", "terraform", "--out", d.terraform.Dir()) } - klog.Info(strings.Join(args, " ")) + if d.KubernetesFeatureGates != "" { + args = appendIfUnset(args, "--kubernetes-feature-gates", d.KubernetesFeatureGates) + } + cmd := exec.Command(args[0], args[1:]...) cmd.SetEnv(d.env()...) @@ -308,11 +322,19 @@ func (d *deployer) zones() ([]string, error) { } // appendIfUnset will append an argument and its value to args if the arg is not already present -// This shouldn't be used for arguments that can be specified multiple times like --set +// This shouldn't be used for arguments that can be specified multiple times except --set func appendIfUnset(args []string, arg, value string) []string { + setFlags := []string{} for _, existingArg := range args { - existingKey := strings.Split(existingArg, "=") - if existingKey[0] == arg { + existingKey := strings.SplitN(existingArg, "=", 2) + if existingKey[0] == "--set" { + if len(existingKey) == 3 { + setFlags = append(setFlags, existingKey[1]) + } + if slices.Contains(setFlags, arg) { + return args + } + } else if existingKey[0] == arg { return args } } diff --git a/tests/e2e/kubetest2-kops/gce/gcs.go b/tests/e2e/kubetest2-kops/gce/gcs.go index 9830ff7d69..45cf93fd2a 100644 --- a/tests/e2e/kubetest2-kops/gce/gcs.go +++ b/tests/e2e/kubetest2-kops/gce/gcs.go @@ -17,8 +17,8 @@ limitations under the License. package gce import ( + "crypto/rand" "encoding/hex" - "math/rand" "os" "strings" diff --git a/tests/e2e/pkg/util/publicip.go b/tests/e2e/pkg/util/publicip.go index aeeed6cc6a..07b70cde94 100644 --- a/tests/e2e/pkg/util/publicip.go +++ b/tests/e2e/pkg/util/publicip.go @@ -25,11 +25,9 @@ import ( "time" ) -const externalIPMetadataURL = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" - var externalIPServiceURLs = []string{ "https://ip.jsb.workers.dev", - "https://v4.ifconfig.co", + "https://ifconfig.co", } // ExternalIPRange returns the CIDR block for the public IP @@ -37,25 +35,18 @@ var externalIPServiceURLs = []string{ func ExternalIPRange() (string, error) { var b bytes.Buffer - err := HTTPGETWithHeaders(externalIPMetadataURL, map[string]string{"Metadata-Flavor": "Google"}, &b) - if err != nil { - // This often fails due to workload identity - log.Printf("failed to get external ip from metadata service: %v", err) - } else if ip := net.ParseIP(strings.TrimSpace(b.String())); ip != nil { - return ip.String() + "/32", nil - } else { - log.Printf("metadata service returned invalid ip %q", b.String()) - } - for attempt := 0; attempt < 5; attempt++ { for _, u := range externalIPServiceURLs { b.Reset() - err = HTTPGETWithHeaders(u, nil, &b) + err := HTTPGETWithHeaders(u, nil, &b) if err != nil { // The external service may well be down log.Printf("failed to get external ip from %s: %v", u, err) } else if ip := net.ParseIP(strings.TrimSpace(b.String())); ip != nil { - return ip.String() + "/32", nil + if ip.To4() != nil { + return ip.String() + "/32", nil + } + return ip.String() + "/128", nil } else { log.Printf("service %s returned invalid ip %q", u, b.String()) }