From 3b06dc9a8cc181e45db0bec250084fc3bc4223c2 Mon Sep 17 00:00:00 2001 From: Evan Anderson Date: Tue, 28 Sep 2021 01:03:14 -0700 Subject: [PATCH] Disable failing integration tests (#4300) * Disable integration tests because they break a lot, per julz * Actually run go and notice extra unused imports * Revert "Disable integration tests because they break a lot, per julz" I took out too much. This reverts commit eca3a2d8ce49ff4de94f3638d3be10c711d6a094. * Re-add simplifed sampleapp used by the consistency checker * Forgot to autoformat after beating things with emacs and grep * Add empty e2e tests * Fix formatting again. `vi` this time --- test/e2e/main.go | 4 +- test/e2e/sampleapp_test.go | 50 --------- test/e2e/sampleapptestbase.go | 197 ---------------------------------- test/flag.go | 28 +---- test/sampleapp/config.go | 45 +------- test/sampleapp/config.yaml | 20 ---- 6 files changed, 9 insertions(+), 335 deletions(-) delete mode 100644 test/e2e/sampleapp_test.go delete mode 100644 test/e2e/sampleapptestbase.go diff --git a/test/e2e/main.go b/test/e2e/main.go index feaf16d53..cf3662481 100644 --- a/test/e2e/main.go +++ b/test/e2e/main.go @@ -14,4 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2etest +// Temporarily have an empty package here until we remove the prow jobs +// https://github.com/knative/test-infra/pull/2891 +package e2e diff --git a/test/e2e/sampleapp_test.go b/test/e2e/sampleapp_test.go deleted file mode 100644 index 32b8ea9eb..000000000 --- a/test/e2e/sampleapp_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// +build e2e - -/* -Copyright 2019 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2etest - -import ( - "testing" - - "github.com/knative/docs/test/sampleapp" - - "github.com/knative/docs/test" -) - -const ( - configFile = "../sampleapp/config.yaml" -) - -// TestSampleApp runs all sample apps from different languages -func TestSampleApp(t *testing.T) { - lcs, err := sampleapp.GetConfigs(configFile) - if nil != err { - t.Fatalf("Failed reading config file %s: '%v'", configFile, err) - } - - allowed := test.GetAllowedLanguages() - for _, lc := range lcs.Languages { - if _, ok := allowed[lc.Language]; len(allowed) > 0 && !ok { - continue - } - lc.UseDefaultIfNotProvided() - t.Run(lc.Language, func(t *testing.T) { - SampleAppTestBase(t, lc, lc.ExpectedOutput) - }) - } -} diff --git a/test/e2e/sampleapptestbase.go b/test/e2e/sampleapptestbase.go deleted file mode 100644 index 97b7d500e..000000000 --- a/test/e2e/sampleapptestbase.go +++ /dev/null @@ -1,197 +0,0 @@ -// +build e2e - -/* -Copyright 2019 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2etest - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "os/exec" - "os/signal" - "path" - "strings" - "testing" - "time" - - "github.com/knative/docs/test/sampleapp" - - "github.com/knative/docs/test" -) - -const ( - servingNamespace = "default" - ingressTimeout = 5 * time.Minute - servingTimeout = 2 * time.Minute - checkInterval = 2 * time.Second -) - -// CleanupOnInterrupt will execute the function cleanup if an interrupt signal is caught -func CleanupOnInterrupt(cleanup func()) { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - go func() { - for range c { - cleanup() - os.Exit(1) - } - }() -} - -func noStderrShell(name string, arg ...string) string { - var void bytes.Buffer - cmd := exec.Command(name, arg...) - cmd.Stderr = &void - out, _ := cmd.Output() - return string(out) -} - -func cleanup(yamlFilePath, workDir string) { - exec.Command("kubectl", "delete", "-f", yamlFilePath).Run() - os.Remove(yamlFilePath) - os.RemoveAll(workDir) -} - -func serviceHostname(appName string) string { - return noStderrShell("kubectl", "get", "rt", appName, "-o", "jsonpath={.status.url}", "-n", servingNamespace) -} - -func ingressAddress(gateway string, addressType string) string { - return noStderrShell("kubectl", "get", "svc", gateway, "-n", "istio-system", - "-o", fmt.Sprintf("jsonpath={.status.loadBalancer.ingress[*]['%s']}", addressType)) -} - -func prepareWorkDir(t *testing.T, srcDir, workDir string, preCommands []sampleapp.Command, copies []string, postCommands []sampleapp.Command) { - t.Log("Prepare source project") - err := os.RemoveAll(workDir) // this function returns nil if path not found - if nil == err { - if _, err = os.Stat(workDir); os.IsNotExist(err) { - err = os.MkdirAll(workDir, 0755) - } - } - if nil != err { - t.Fatalf("Failed preparing local artifacts directory: %v", err) - } - - for _, c := range preCommands { - c.Run(t) - } - for _, f := range copies { - src := path.Join(srcDir, f) - dst := path.Join(workDir, f) - dstDir, _ := path.Split(dst) - noStderrShell("mkdir", "-p", dstDir) - if output, err := exec.Command("cp", src, dst).CombinedOutput(); err != nil { - t.Fatalf("Error copying: '%s' to '%s' -err: '%v'", src, dst, strings.TrimSpace(string(output))) - } - } - for _, c := range postCommands { - c.Run(t) - } -} - -func pushDockerImage(t *testing.T, imagePath, workDir string) { - t.Logf("Pushing docker image to: '%s'", imagePath) - if output, err := exec.Command("docker", "build", "-t", imagePath, workDir).CombinedOutput(); err != nil { - t.Fatalf("Error building docker image: %v", strings.TrimSpace(string(output))) - } - if output, err := exec.Command("docker", "push", imagePath).CombinedOutput(); err != nil { - t.Fatalf("Error pushing docker image: %v", strings.TrimSpace(string(output))) - } -} - -func deploy(t *testing.T, yamlFilePath, yamlImagePlaceholder, imagePath string) { - t.Log("Creating manifest") - // Populate manifets file with the real path to the container - yamlBytes, err := ioutil.ReadFile(yamlFilePath) - if err != nil { - t.Fatalf("Failed to read file %s: %v", yamlFilePath, err) - } - content := strings.Replace(string(yamlBytes), yamlImagePlaceholder, imagePath, -1) - if err = ioutil.WriteFile(yamlFilePath, []byte(content), 0644); err != nil { - t.Fatalf("Failed to write new manifest: %v", err) - } - - t.Logf("Deploying using kubectl and using manifest file %q", yamlFilePath) - // Deploy using kubectl - if output, err := exec.Command("kubectl", "apply", "-f", yamlFilePath).CombinedOutput(); err != nil { - t.Fatalf("Error running kubectl: %v", strings.TrimSpace(string(output))) - } -} - -func checkDeployment(t *testing.T, appName, expectedOutput string) { - t.Log("Waiting for ingress to come up") - gateway := "istio-ingressgateway" - // Wait for ingress to come up - ingressAddr := "" - serviceHost := "" - timeout := ingressTimeout - for (ingressAddr == "" || serviceHost == "") && timeout >= 0 { - if serviceHost == "" { - serviceHost = serviceHostname(appName) - } - if ingressAddr = ingressAddress(gateway, "ip"); ingressAddr == "" { - ingressAddr = ingressAddress(gateway, "hostname") - } - timeout -= checkInterval - time.Sleep(checkInterval) - } - if ingressAddr == "" || serviceHost == "" { - // serviceHost or ingressAddr might contain a useful error, dump them. - t.Fatalf("Ingress not found (ingress='%s', host='%s')", ingressAddr, serviceHost) - } - t.Logf("Curling %s/%s", ingressAddr, serviceHost) - - serviceHost = strings.Replace(serviceHost, "http://", "", 1) - outputString := "" - timeout = servingTimeout - for outputString != expectedOutput && timeout >= 0 { - output, err := exec.Command("curl", "--header", "Host:"+serviceHost, "http://"+ingressAddr).Output() - errorString := "none" - if err != nil { - errorString = err.Error() - } - outputString = strings.TrimSpace(string(output)) - t.Logf("App replied with '%s' (error: %s)", outputString, errorString) - timeout -= checkInterval - time.Sleep(checkInterval) - } - - if outputString != expectedOutput { - t.Fatal("Timeout waiting for app to start serving") - } - t.Log("App is serving") -} - -// SampleAppTestBase tests individual sample app -func SampleAppTestBase(t *testing.T, lc sampleapp.LanguageConfig, expectedOutput string) { - t.Parallel() - imagePath := test.ImagePath(lc.AppName) - yamlFilePath := path.Join(lc.WorkDir, "service.yaml") - - CleanupOnInterrupt(func() { cleanup(yamlFilePath, lc.WorkDir) }) - defer cleanup(yamlFilePath, lc.WorkDir) - - prepareWorkDir(t, lc.SrcDir, lc.WorkDir, lc.PreCommands, lc.Copies, lc.PostCommands) - pushDockerImage(t, imagePath, lc.WorkDir) - - // Deploy and test - deploy(t, yamlFilePath, lc.YamlImagePlaceholder, imagePath) - checkDeployment(t, lc.AppName, expectedOutput) -} diff --git a/test/flag.go b/test/flag.go index 61889bc1c..0b0476528 100644 --- a/test/flag.go +++ b/test/flag.go @@ -18,8 +18,6 @@ package test import ( "flag" - "fmt" - "os" "strings" ) @@ -29,40 +27,16 @@ var Flags = initializeFlags() // EnvironmentFlags define the flags that are needed to run the e2e tests. type EnvironmentFlags struct { - Cluster string // K8s cluster (defaults to cluster in kubeconfig) - LogVerbose bool // Enable verbose logging - DockerRepo string // Docker repo (defaults to $KO_DOCKER_REPO) - EmitMetrics bool // Emit metrics - Tag string // Docker image tag - Languages string // Allowed languages to run + Languages string // Allowed languages to run } func initializeFlags() *EnvironmentFlags { var f EnvironmentFlags - flag.StringVar(&f.Cluster, "cluster", "", - "Provide the cluster to test against. Defaults to the current cluster in kubeconfig.") - - flag.BoolVar(&f.LogVerbose, "logverbose", false, - "Set this flag to true if you would like to see verbose logging.") - - flag.BoolVar(&f.EmitMetrics, "emitmetrics", false, - "Set this flag to true if you would like tests to emit metrics, e.g. latency of resources being realized in the system.") - - flag.StringVar(&f.DockerRepo, "dockerrepo", os.Getenv("KO_DOCKER_REPO"), - "Provide the uri of the docker repo you have uploaded the test image to using `uploadtestimage.sh`. Defaults to $KO_DOCKER_REPO") - - flag.StringVar(&f.Tag, "tag", "latest", "Provide the version tag for the test images.") - flag.StringVar(&f.Languages, "languages", "", "Comma separated languages to run e2e test on.") return &f } -// ImagePath is a helper function to prefix image name with repo and suffix with tag -func ImagePath(name string) string { - return fmt.Sprintf("%s/%s:%s", Flags.DockerRepo, name, Flags.Tag) -} - // GetAllowedLanguages is a helper function to return a map of allowed languages based on Languages filter func GetAllowedLanguages() map[string]bool { allowed := make(map[string]bool) diff --git a/test/sampleapp/config.go b/test/sampleapp/config.go index 15930c3bc..2a1659ca5 100644 --- a/test/sampleapp/config.go +++ b/test/sampleapp/config.go @@ -19,19 +19,13 @@ package sampleapp import ( "fmt" "io/ioutil" - "os/exec" - "strings" - "testing" yaml "gopkg.in/yaml.v2" ) const ( // using these defaults if not provided, see useDefaultIfNotProvided function below - defaultSrcDir = "../../docs/serving/samples/hello-world/helloworld-%s" - defaultWorkDir = "helloworld-%s_tmp" - defaultAppName = "helloworld-%s" - defaultYamlImagePlaceHolder = "docker.io/{username}/helloworld-%s" + defaultSrcDir = "../../docs/serving/samples/hello-world/helloworld-%s" // ActionMsg serves as documentation purpose, which will be referenced for // clearly displaying error messages. @@ -51,45 +45,16 @@ type AllConfigs struct { // LanguageConfig contains all information for building/deploying an app type LanguageConfig struct { - Language string `yaml:"language"` - ExpectedOutput string `yaml:"expectedOutput"` - SrcDir string `yaml:"srcDir"` // Directory contains sample code - WorkDir string `yaml:"workDir"` // Temp work directory - AppName string `yaml:"appName"` - YamlImagePlaceholder string `yaml:"yamlImagePlaceholder"` // Token to be replaced by real docker image URI - PreCommands []Command `yaml:"preCommands"` // Commands to be ran before copying - Copies []string `yaml:"copies"` // Files to be copied from SrcDir to WorkDir - PostCommands []Command `yaml:"postCommands"` // Commands to be ran after copying + Language string `yaml:"language"` + SrcDir string `yaml:"srcDir"` // Directory contains sample code + Copies []string `yaml:"copies"` // Files to be copied by the user from SrcDir } -// Command contains shell commands -type Command struct { - Exec string `yaml:"exec"` - Args string `yaml:"args"` -} - -// UseDefaultIfNotProvided sets default value to SrcDir, WorkDir, AppName, and YamlImagePlaceholder if not provided +// UseDefaultIfNotProvided sets default value of SrcDir if not provided func (lc *LanguageConfig) UseDefaultIfNotProvided() { if "" == lc.SrcDir { lc.SrcDir = fmt.Sprintf(defaultSrcDir, lc.Language) } - if "" == lc.WorkDir { - lc.WorkDir = fmt.Sprintf(defaultWorkDir, lc.Language) - } - if "" == lc.AppName { - lc.AppName = fmt.Sprintf(defaultAppName, lc.Language) - } - if "" == lc.YamlImagePlaceholder { - lc.YamlImagePlaceholder = fmt.Sprintf(defaultYamlImagePlaceHolder, lc.Language) - } -} - -// Run runs command and fail if it failed -func (c *Command) Run(t *testing.T) { - args := strings.Split(c.Args, " ") - if output, err := exec.Command(c.Exec, args...).CombinedOutput(); err != nil { - t.Fatalf("Error executing: '%s' '%s' -err: '%v'", c.Exec, c.Args, strings.TrimSpace(string(output))) - } } // GetConfigs parses a config yaml file and return AllConfigs struct diff --git a/test/sampleapp/config.yaml b/test/sampleapp/config.yaml index 402b38121..f23dfd7a3 100644 --- a/test/sampleapp/config.yaml +++ b/test/sampleapp/config.yaml @@ -1,63 +1,43 @@ languages: - language: "java-spring" - expectedOutput: "Hello Spring Boot Sample v1!" - preCommands: - - exec: "unzip" - args: "../sampleapp/helloworld-java-spring.zip -d helloworld-java-spring_tmp" copies: - "/src/main/java/com/example/helloworld/HelloworldApplication.java" - "service.yaml" - "Dockerfile" - language: "go" - expectedOutput: "Hello Go Sample v1!" - preCommands: - - exec: "cp" - args: "../../docs/serving/samples/hello-world/helloworld-go/go.mod helloworld-go_tmp/go.mod" copies: - "helloworld.go" - "service.yaml" - "Dockerfile" - language: "kotlin" - expectedOutput: "Hello Kotlin Sample v1!" copies: - "/src/main/kotlin/com/example/hello/Main.kt" - "build.gradle" - "service.yaml" - "Dockerfile" - language: "nodejs" - expectedOutput: "Hello Node.js Sample v1!" - preCommands: - - exec: "cp" - args: "../../docs/serving/samples/hello-world/helloworld-nodejs/package-lock.json helloworld-nodejs_tmp/package-lock.json" copies: - "index.js" - "package.json" - "service.yaml" - "Dockerfile" - language: "php" - expectedOutput: "Hello PHP Sample v1!" copies: - "index.php" - "service.yaml" - "Dockerfile" - language: "python" - expectedOutput: "Hello Python Sample v1!" copies: - "app.py" - "service.yaml" - "Dockerfile" - language: "ruby" - expectedOutput: "Hello Ruby Sample v1!" - preCommands: - - exec: "cp" - args: "../../docs/serving/samples/hello-world/helloworld-ruby/Gemfile.lock helloworld-ruby_tmp/Gemfile.lock" copies: - "app.rb" - "Gemfile" - "service.yaml" - "Dockerfile" - language: "shell" - expectedOutput: "Hello Shell Sample v1!" copies: - "script.sh" - "service.yaml"