kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15900 from borg-land/fix-e2e-testing 

Fix e2e testing logic
This commit is contained in:
Kubernetes Prow Robot 2023-09-22 13:38:49 -07:00 committed by GitHub
parent 0a0f938d91 6bdefda4ec
commit 06d1d8965a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 44 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/e2e/e2e.mk
View File

@ -18,7 +18,6 @@ GOARCH ?= $(shell go env GOARCH)
.PHONY: test-e2e-install
test-e2e-install:
cd $(KOPS_ROOT)/tests/e2e && \
go install sigs.k8s.io/kubetest2 && \
go install ./kubetest2-tester-kops && \
go install ./kubetest2-kops

2
tests/e2e/go.mod
View File

@ -17,6 +17,7 @@ require (
k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible
k8s.io/klog/v2 v2.100.1
k8s.io/kops v1.24.1
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
sigs.k8s.io/boskos v0.0.0-20220704141725-37bd9bb41b86
sigs.k8s.io/kubetest2 v0.0.0-20220801170629-1284e5ada592
sigs.k8s.io/yaml v1.3.0
@ -139,7 +140,6 @@ require (
k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect
k8s.io/release v0.7.1-0.20210204090829-09fb5e3883b8 // indirect
k8s.io/test-infra v0.0.0-20210730160938-8ad9b8c53bd8 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
)

15
tests/e2e/kubetest2-kops/deployer/deployer.go
View File

@ -48,13 +48,14 @@ type deployer struct {
KopsBaseURL string `flag:"-"`
PublishVersionMarker string `flag:"publish-version-marker" desc:"The GCS path to which the --kops-version-marker is uploaded if the tests pass"`
ClusterName string `flag:"cluster-name" desc:"The FQDN to use for the cluster name"`
CloudProvider string `flag:"cloud-provider" desc:"Which cloud provider to use"`
GCPProject string `flag:"gcp-project" desc:"Which GCP Project to use when --cloud-provider=gce"`
Env []string `flag:"env" desc:"Additional env vars to set for kops commands in NAME=VALUE format"`
CreateArgs string `flag:"create-args" desc:"Extra space-separated arguments passed to 'kops create cluster'"`
KopsBinaryPath string `flag:"kops-binary-path" desc:"The path to kops executable used for testing"`
createBucket bool `flag:"-"`
ClusterName string `flag:"cluster-name" desc:"The FQDN to use for the cluster name"`
CloudProvider string `flag:"cloud-provider" desc:"Which cloud provider to use"`
GCPProject string `flag:"gcp-project" desc:"Which GCP Project to use when --cloud-provider=gce"`
Env []string `flag:"env" desc:"Additional env vars to set for kops commands in NAME=VALUE format"`
CreateArgs string `flag:"create-args" desc:"Extra space-separated arguments passed to 'kops create cluster'"`
KopsBinaryPath string `flag:"kops-binary-path" desc:"The path to kops executable used for testing"`
KubernetesFeatureGates string `flag:"kubernetes-feature-gates" desc:"Feature Gates to enable on Kubernetes components"`
createBucket bool `flag:"-"`
// ControlPlaneCount specifies the number of VMs in the control-plane.
ControlPlaneCount int `flag:"control-plane-count" desc:"Number of control-plane instances"`

5
tests/e2e/kubetest2-kops/deployer/dumplogs.go
View File

@ -281,8 +281,3 @@ func findControlPlaneIPUser(dump resources.Dump) (string, string, bool) {
klog.Warning("ControlPlane instance not found from kops toolbox dump")
return "", "", false
}
func runWithOutput(cmd exec.Cmd) error {
exec.InheritOutput(cmd)
return cmd.Run()
}

34
tests/e2e/kubetest2-kops/deployer/up.go
View File

@ -32,6 +32,7 @@ import (
"k8s.io/kops/tests/e2e/pkg/kops"
"k8s.io/kops/tests/e2e/pkg/util"
"k8s.io/kops/tests/e2e/pkg/version"
"k8s.io/utils/strings/slices"
"sigs.k8s.io/kubetest2/pkg/exec"
)
@ -152,21 +153,31 @@ func (d *deployer) createCluster(zones []string, adminAccess string, yes bool) e
args = appendIfUnset(args, "--master-volume-size", "48")
args = appendIfUnset(args, "--node-count", "4")
args = appendIfUnset(args, "--node-volume-size", "48")
args = appendIfUnset(args, "--set", adminAccess)
args = appendIfUnset(args, "--zones", strings.Join(zones, ","))
switch d.CloudProvider {
case "aws":
if isArm {
args = appendIfUnset(args, "--master-size", "c6g.large")
args = appendIfUnset(args, "--node-size", "c6g.large")
} else {
args = appendIfUnset(args, "--master-size", "c5.large")
}
case "gce":
args = appendIfUnset(args, "--master-size", "e2-standard-2")
if isArm {
args = appendIfUnset(args, "--master-size", "t2a-standard-2")
args = appendIfUnset(args, "--node-size", "t2a-standard-2")
} else {
args = appendIfUnset(args, "--master-size", "e2-standard-2")
args = appendIfUnset(args, "--node-size", "e2-standard-2")
}
if d.GCPProject != "" {
args = appendIfUnset(args, "--project", d.GCPProject)
}
// set some sane default e2e testing behaviour on gce
args = appendIfUnset(args, "--gce-service-account", "default")
args = appendIfUnset(args, "--networking", "kubenet")
// We used to set the --vpc flag to split clusters into different networks, this is now the default.
// args = appendIfUnset(args, "--vpc", strings.Split(d.ClusterName, ".")[0])
case "digitalocean":
@ -178,7 +189,10 @@ func (d *deployer) createCluster(zones []string, adminAccess string, yes bool) e
args = append(args, "--target", "terraform", "--out", d.terraform.Dir())
}
klog.Info(strings.Join(args, " "))
if d.KubernetesFeatureGates != "" {
args = appendIfUnset(args, "--kubernetes-feature-gates", d.KubernetesFeatureGates)
}
cmd := exec.Command(args[0], args[1:]...)
cmd.SetEnv(d.env()...)
@ -308,11 +322,19 @@ func (d *deployer) zones() ([]string, error) {
}
// appendIfUnset will append an argument and its value to args if the arg is not already present
// This shouldn't be used for arguments that can be specified multiple times like --set
// This shouldn't be used for arguments that can be specified multiple times except --set
func appendIfUnset(args []string, arg, value string) []string {
setFlags := []string{}
for _, existingArg := range args {
existingKey := strings.Split(existingArg, "=")
if existingKey[0] == arg {
existingKey := strings.SplitN(existingArg, "=", 2)
if existingKey[0] == "--set" {
if len(existingKey) == 3 {
setFlags = append(setFlags, existingKey[1])
}
if slices.Contains(setFlags, arg) {
return args
}
} else if existingKey[0] == arg {
return args
}
}

2
tests/e2e/kubetest2-kops/gce/gcs.go
View File

@ -17,8 +17,8 @@ limitations under the License.
package gce
import (
"crypto/rand"
"encoding/hex"
"math/rand"
"os"
"strings"

21
tests/e2e/pkg/util/publicip.go
View File

@ -25,11 +25,9 @@ import (
"time"
)
const externalIPMetadataURL = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"
var externalIPServiceURLs = []string{
"https://ip.jsb.workers.dev",
"https://v4.ifconfig.co",
"https://ifconfig.co",
}
// ExternalIPRange returns the CIDR block for the public IP
@ -37,25 +35,18 @@ var externalIPServiceURLs = []string{
func ExternalIPRange() (string, error) {
var b bytes.Buffer
err := HTTPGETWithHeaders(externalIPMetadataURL, map[string]string{"Metadata-Flavor": "Google"}, &b)
if err != nil {
// This often fails due to workload identity
log.Printf("failed to get external ip from metadata service: %v", err)
} else if ip := net.ParseIP(strings.TrimSpace(b.String())); ip != nil {
return ip.String() + "/32", nil
} else {
log.Printf("metadata service returned invalid ip %q", b.String())
}
for attempt := 0; attempt < 5; attempt++ {
for _, u := range externalIPServiceURLs {
b.Reset()
err = HTTPGETWithHeaders(u, nil, &b)
err := HTTPGETWithHeaders(u, nil, &b)
if err != nil {
// The external service may well be down
log.Printf("failed to get external ip from %s: %v", u, err)
} else if ip := net.ParseIP(strings.TrimSpace(b.String())); ip != nil {
return ip.String() + "/32", nil
if ip.To4() != nil {
return ip.String() + "/32", nil
}
return ip.String() + "/128", nil
} else {
log.Printf("service %s returned invalid ip %q", u, b.String())
}