From 17468d496f7a8d720d1866cbcfaaddeee6e413cd Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Sat, 9 Jan 2021 23:29:20 -0600 Subject: [PATCH] Kubetest2 - use our own tester that wraps kubetest2's ginkgo tester This allows us to share tester flags (package version and bucket) for downloading kubectl while passing them to the ginkgo tester --- Makefile | 4 +- tests/e2e/kubetest2-tester-kops/main.go | 25 +++ tests/e2e/pkg/tester/kubectl.go | 192 ++++++++++++++++++++++++ tests/e2e/pkg/tester/tester.go | 66 ++++++++ tests/e2e/scenarios/upgrade/run-test | 4 +- 5 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/kubetest2-tester-kops/main.go create mode 100644 tests/e2e/pkg/tester/kubectl.go create mode 100644 tests/e2e/pkg/tester/tester.go diff --git a/Makefile b/Makefile index a88492eb84..f12f389d93 100644 --- a/Makefile +++ b/Makefile @@ -190,7 +190,7 @@ test-e2e: cd /home/prow/go/src/k8s.io/kops/tests/e2e && \ export GO111MODULE=on && \ go get sigs.k8s.io/kubetest2@latest && \ - go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest && \ + go install ./kubetest2-tester-kops && \ go install ./kubetest2-kops kubetest2 kops \ -v 2 \ @@ -198,7 +198,7 @@ test-e2e: --cloud-provider=aws \ --kops-binary-path=/home/prow/go/src/k8s.io/kops/bazel-bin/cmd/kops/linux-amd64/kops \ --kubernetes-version=v1.19.4 \ - --test=ginkgo \ + --test=kops \ -- \ --test-package-version=v1.19.4 \ --parallel 25 \ diff --git a/tests/e2e/kubetest2-tester-kops/main.go b/tests/e2e/kubetest2-tester-kops/main.go new file mode 100644 index 0000000000..f1f2290890 --- /dev/null +++ b/tests/e2e/kubetest2-tester-kops/main.go @@ -0,0 +1,25 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + tester "k8s.io/kops/tests/e2e/pkg/tester" +) + +func main() { + tester.Main() +} diff --git a/tests/e2e/pkg/tester/kubectl.go b/tests/e2e/pkg/tester/kubectl.go new file mode 100644 index 0000000000..162804dd9f --- /dev/null +++ b/tests/e2e/pkg/tester/kubectl.go @@ -0,0 +1,192 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tester + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "runtime" + "strings" + + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/exec" +) + +// AcquireKubectl obtains kubectl and places it in a temporary directory +func (t *Tester) AcquireKubectl() (string, error) { + // first, get the name of the latest release (e.g. v1.20.0-alpha.0) + if t.Ginkgo.TestPackageVersion == "" { + cmd := exec.Command( + "gsutil", + "cat", + fmt.Sprintf("gs://%s/%s/latest.txt", t.Ginkgo.TestPackageBucket, t.Ginkgo.TestPackageDir), + ) + lines, err := exec.OutputLines(cmd) + if err != nil { + return "", fmt.Errorf("failed to get latest release name: %s", err) + } + if len(lines) == 0 { + return "", fmt.Errorf("getting latest release name had no output") + } + t.Ginkgo.TestPackageVersion = lines[0] + + klog.V(1).Infof("Kubectl package version was not specified. Defaulting to latest: %s", t.Ginkgo.TestPackageVersion) + } + + clientTar := fmt.Sprintf("kubernetes-client-%s-%s.tar.gz", runtime.GOOS, runtime.GOARCH) + + downloadDir, err := os.UserCacheDir() + if err != nil { + return "", fmt.Errorf("failed to get user cache directory: %v", err) + } + + downloadPath := filepath.Join(downloadDir, clientTar) + + if err := t.ensureClientTar(downloadPath, clientTar); err != nil { + return "", err + } + + return t.extractBinaries(downloadPath) +} + +func (t *Tester) extractBinaries(downloadPath string) (string, error) { + // finally, search for the client package and extract it + f, err := os.Open(downloadPath) + if err != nil { + return "", fmt.Errorf("failed to open downloaded tar at %s: %s", downloadPath, err) + } + defer f.Close() + gzf, err := gzip.NewReader(f) + if err != nil { + return "", fmt.Errorf("failed to create gzip reader: %s", err) + } + tarReader := tar.NewReader(gzf) + + kubectlDir, err := ioutil.TempDir("", "kubectl") + if err != nil { + return "", err + } + + // this is the expected path of the package inside the tar + // it will be extracted to kubectlDir in the loop + kubectlPackagePath := "kubernetes/client/bin/kubectl" + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return "", fmt.Errorf("error during tar read: %s", err) + } + + if header.Name == kubectlPackagePath { + kubectlPath := path.Join(kubectlDir, "kubectl") + outFile, err := os.Create(kubectlPath) + if err != nil { + return "", fmt.Errorf("error creating file at %s: %s", kubectlPath, err) + } + defer outFile.Close() + + if err := outFile.Chmod(0700); err != nil { + return "", fmt.Errorf("failed to make %s executable: %s", kubectlPath, err) + } + + if _, err := io.Copy(outFile, tarReader); err != nil { + return "", fmt.Errorf("error reading data from tar with header name %s: %s", header.Name, err) + } + return kubectlPath, nil + } + } + return "", fmt.Errorf("failed to find %s in %s", kubectlPackagePath, downloadPath) +} + +// ensureClientTar checks if the kubernetes client tarball already exists +// and verifies the hashes +// else downloads it from GCS +func (t *Tester) ensureClientTar(downloadPath, clientTar string) error { + if _, err := os.Stat(downloadPath); err == nil { + klog.V(0).Infof("Found existing tar at %v", downloadPath) + if err := t.compareSHA(downloadPath, clientTar); err == nil { + klog.V(0).Infof("Validated hash for existing tar at %v", downloadPath) + return nil + } + klog.Warning(err) + } + + cmd := exec.Command("gsutil", "cp", + fmt.Sprintf( + "gs://%s/%s/%s/%s", + t.Ginkgo.TestPackageBucket, + t.Ginkgo.TestPackageDir, + t.Ginkgo.TestPackageVersion, + clientTar, + ), + downloadPath, + ) + exec.InheritOutput(cmd) + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to download release tar %s for release %s: %s", clientTar, t.Ginkgo.TestPackageVersion, err) + } + return nil +} + +func (t *Tester) compareSHA(downloadPath string, clientTar string) error { + cmd := exec.Command("gsutil", "cat", + fmt.Sprintf( + "gs://%s/%s/%s/%s", + t.Ginkgo.TestPackageBucket, + t.Ginkgo.TestPackageDir, + t.Ginkgo.TestPackageVersion, + clientTar+".sha256", + ), + ) + expectedSHABytes, err := exec.Output(cmd) + if err != nil { + return fmt.Errorf("failed to get sha256 for release tar %s for release %s: %s", clientTar, t.Ginkgo.TestPackageVersion, err) + } + expectedSHA := strings.TrimSuffix(string(expectedSHABytes), "\n") + actualSHA, err := sha256sum(downloadPath) + if err != nil { + return fmt.Errorf("failed to compute sha256 for %q: %v", downloadPath, err) + } + if actualSHA != expectedSHA { + return fmt.Errorf("sha256 does not match") + } + return nil +} + +func sha256sum(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/tests/e2e/pkg/tester/tester.go b/tests/e2e/pkg/tester/tester.go new file mode 100644 index 0000000000..36dcfe80aa --- /dev/null +++ b/tests/e2e/pkg/tester/tester.go @@ -0,0 +1,66 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tester + +import ( + "fmt" + "os" + + "k8s.io/klog/v2" + + "sigs.k8s.io/kubetest2/pkg/testers/ginkgo" +) + +// Tester wraps kubetest2's ginkgo tester with additional functionality +type Tester struct { + Ginkgo *ginkgo.Tester +} + +// Test runs the test +func (t *Tester) Test() error { + if err := t.pretestSetup(); err != nil { + return err + } + return t.Ginkgo.Test() +} + +func (t *Tester) pretestSetup() error { + kubectlPath, err := t.AcquireKubectl() + if err != nil { + return fmt.Errorf("failed to get kubectl package from published releases: %s", err) + } + existingPath := os.Getenv("PATH") + os.Setenv("PATH", fmt.Sprintf("%v:%v", kubectlPath, existingPath)) + return nil +} + +func (t *Tester) Execute() error { + return t.Ginkgo.Execute() +} + +func NewDefaultTester() *Tester { + return &Tester{ + Ginkgo: ginkgo.NewDefaultTester(), + } +} + +func Main() { + t := NewDefaultTester() + if err := t.Execute(); err != nil { + klog.Fatalf("failed to run ginkgo tester: %v", err) + } +} diff --git a/tests/e2e/scenarios/upgrade/run-test b/tests/e2e/scenarios/upgrade/run-test index 051cce18d2..6a8b8dd960 100755 --- a/tests/e2e/scenarios/upgrade/run-test +++ b/tests/e2e/scenarios/upgrade/run-test @@ -27,10 +27,10 @@ KUBETEST2_COMMON_ARGS="${KUBETEST2_COMMON_ARGS} --admin-access=${ADMIN_ACCESS:-} export GO111MODULE=on go get sigs.k8s.io/kubetest2@latest -go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest cd ${REPO_ROOT}/tests/e2e go install ./kubetest2-kops +go install ./kubetest2-tester-kops kubetest2 kops ${KUBETEST2_COMMON_ARGS} --build --kops-root=${REPO_ROOT} --stage-location=${STAGE_LOCATION:-} @@ -59,7 +59,7 @@ fi kubetest2 kops ${KUBETEST2_COMMON_ARGS} \ --cloud-provider=${CLOUD_PROVIDER} \ - --test=ginkgo \ + --test=kops \ -- \ --test-package-version=v1.19.4 \ --parallel 25 \