Merge pull request #10504 from rifelpet/kubetest2-test

Run k/k's e2e suite via new kubetest2 make target

Makefile

@@ -26,7 +26,7 @@ UNIQUE:=$(shell date +%s)
 BUILD=$(KOPS_ROOT)/.build
 LOCAL=$(BUILD)/local
 BINDATA_TARGETS=upup/models/bindata.go
-ARTIFACTS=$(BUILD)/artifacts
+ARTIFACTS?=$(BUILD)/artifacts
 DIST=$(BUILD)/dist
 IMAGES=$(DIST)/images
 CHANNELS=$(LOCAL)/channels
@@ -185,6 +185,24 @@ test: ${BINDATA_TARGETS}  # Run tests locally
 test-windows: ${BINDATA_TARGETS}  # Run tests locally
 	go test -v $(go list ./... | grep -v /nodeup/)
 
+.PHONY: test-e2e
+test-e2e:
+	cd /home/prow/go/src/k8s.io/kops/tests/e2e && \
+		export GO111MODULE=on && \
+		go get sigs.k8s.io/kubetest2@latest && \
+		go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest && \
+		go install ./kubetest2-kops
+	kubetest2 kops \
+		-v 2 \
+		--build --up --down \
+		--cloud-provider=aws \
+		--kops-binary-path=/home/prow/go/src/k8s.io/kops/bazel-bin/cmd/kops/linux-amd64/kops \
+		--test=ginkgo \
+		-- \
+		--test-package-version=v1.19.4 \
+		--parallel 25 \
+		--skip-regex="\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[HPA\]|Dashboard|Services.*functioning.*NodePort"
+
 .PHONY: ${DIST}/linux/amd64/nodeup
 ${DIST}/linux/amd64/nodeup: ${BINDATA_TARGETS}
 	mkdir -p ${DIST}

tests/e2e/go.mod

@@ -5,6 +5,6 @@ go 1.15
 require (
 	github.com/octago/sflags v0.2.0
 	github.com/spf13/pflag v1.0.5
-	k8s.io/klog/v2 v2.3.0
+	k8s.io/klog/v2 v2.4.0
-	sigs.k8s.io/kubetest2 v0.0.0-20200930205053-4e57931be178
+	sigs.k8s.io/kubetest2 v0.0.0-20201130212850-d9dad7c8699c
 )

tests/e2e/kubetest2-kops/deployer/common.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 
 	"k8s.io/klog/v2"
 )
@@ -137,3 +138,22 @@ func stateStore(cloudProvider string) string {
 	}
 	return ss
 }
+
+// the default is $ARTIFACTS if set, otherwise ./_artifacts
+// constructed as an absolute path to help the ginkgo tester because
+// for some reason it needs an absolute path to the kubeconfig
+func defaultArtifactsDir() (string, error) {
+	if path, set := os.LookupEnv("ARTIFACTS"); set {
+		absPath, err := filepath.Abs(path)
+		if err != nil {
+			return "", fmt.Errorf("failed to convert filepath from $ARTIFACTS (%s) to absolute path: %s", path, err)
+		}
+		return absPath, nil
+	}
+
+	absPath, err := filepath.Abs("_artifacts")
+	if err != nil {
+		return "", fmt.Errorf("when constructing default artifacts dir, failed to get absolute path: %s", err)
+	}
+	return absPath, nil
+}

tests/e2e/kubetest2-kops/deployer/deployer.go

@@ -51,6 +51,8 @@ type deployer struct {
 	SSHPublicKeyPath  string   `flag:"ssh-public-key" desc:"The path to the public key passed to the cloud provider"`
 	SSHUser           []string `flag:"ssh-user" desc:"The SSH users to use for SSH access to instances"`
 
+	ArtifactsDir string `flag:"-"`
+
 	BuildOptions *builder.BuildOptions
 }
 
@@ -64,11 +66,6 @@ func (d *deployer) Provider() string {
 	return Name
 }
 
-func (d *deployer) DumpClusterLogs() error {
-	klog.Warning("DumpClusterLogs is not implemented")
-	return nil
-}
-
 // New implements deployer.New for kops
 func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
 	// create a deployer object and set fields that are not flag controlled
@@ -77,6 +74,12 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
 		BuildOptions:  &builder.BuildOptions{},
 	}
 
+	dir, err := defaultArtifactsDir()
+	if err != nil {
+		klog.Fatalf("unable to determine artifacts directory: %v", err)
+	}
+	d.ArtifactsDir = dir
+
 	// register flags
 	fs := bindFlags(d)
 

tests/e2e/kubetest2-kops/deployer/down.go

@@ -27,6 +27,10 @@ func (d *deployer) Down() error {
 	if err := d.init(); err != nil {
 		return err
 	}
+	if err := d.DumpClusterLogs(); err != nil {
+		klog.Warningf("Dumping cluster logs at the start of Down() failed: %s", err)
+	}
+
 	args := []string{
 		d.KopsBinaryPath, "delete", "cluster",
 		"--name", d.ClusterName,

tests/e2e/kubetest2-kops/deployer/dumplogs.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deployer
+
+import (
+	"strings"
+
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/kubetest2/pkg/exec"
+)
+
+func (d *deployer) DumpClusterLogs() error {
+
+	args := []string{
+		d.KopsBinaryPath, "toolbox", "dump",
+		"--name", d.ClusterName,
+		"--dir", d.ArtifactsDir,
+		"--private-key", d.SSHPrivateKeyPath,
+	}
+	klog.Info(strings.Join(args, " "))
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.SetEnv(d.env()...)
+	if err := runWithOutput(cmd); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func runWithOutput(cmd exec.Cmd) error {
+	exec.InheritOutput(cmd)
+	return cmd.Run()
+}