Disable failing integration tests (#4300)

* Disable integration tests because they break a lot, per julz

* Actually run go and notice extra unused imports

* Revert "Disable integration tests because they break a lot, per julz"

I took out too much.

This reverts commit eca3a2d8ce.

* Re-add simplifed sampleapp used by the consistency checker

* Forgot to autoformat after beating things with emacs and grep

* Add empty e2e tests

* Fix formatting again. `vi` this time

test/e2e/main.go

@@ -14,4 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2etest
+// Temporarily have an empty package here until we remove the prow jobs
+// https://github.com/knative/test-infra/pull/2891
+package e2e

test/e2e/sampleapp_test.go

@@ -1,50 +0,0 @@
-// +build e2e
-
-/*
-Copyright 2019 The Knative Authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2etest
-
-import (
-	"testing"
-
-	"github.com/knative/docs/test/sampleapp"
-
-	"github.com/knative/docs/test"
-)
-
-const (
-	configFile = "../sampleapp/config.yaml"
-)
-
-// TestSampleApp runs all sample apps from different languages
-func TestSampleApp(t *testing.T) {
-	lcs, err := sampleapp.GetConfigs(configFile)
-	if nil != err {
-		t.Fatalf("Failed reading config file %s: '%v'", configFile, err)
-	}
-
-	allowed := test.GetAllowedLanguages()
-	for _, lc := range lcs.Languages {
-		if _, ok := allowed[lc.Language]; len(allowed) > 0 && !ok {
-			continue
-		}
-		lc.UseDefaultIfNotProvided()
-		t.Run(lc.Language, func(t *testing.T) {
-			SampleAppTestBase(t, lc, lc.ExpectedOutput)
-		})
-	}
-}

test/e2e/sampleapptestbase.go

@@ -1,197 +0,0 @@
-// +build e2e
-
-/*
-Copyright 2019 The Knative Authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2etest
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"os/signal"
-	"path"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/knative/docs/test/sampleapp"
-
-	"github.com/knative/docs/test"
-)
-
-const (
-	servingNamespace = "default"
-	ingressTimeout   = 5 * time.Minute
-	servingTimeout   = 2 * time.Minute
-	checkInterval    = 2 * time.Second
-)
-
-// CleanupOnInterrupt will execute the function cleanup if an interrupt signal is caught
-func CleanupOnInterrupt(cleanup func()) {
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt)
-	go func() {
-		for range c {
-			cleanup()
-			os.Exit(1)
-		}
-	}()
-}
-
-func noStderrShell(name string, arg ...string) string {
-	var void bytes.Buffer
-	cmd := exec.Command(name, arg...)
-	cmd.Stderr = &void
-	out, _ := cmd.Output()
-	return string(out)
-}
-
-func cleanup(yamlFilePath, workDir string) {
-	exec.Command("kubectl", "delete", "-f", yamlFilePath).Run()
-	os.Remove(yamlFilePath)
-	os.RemoveAll(workDir)
-}
-
-func serviceHostname(appName string) string {
-	return noStderrShell("kubectl", "get", "rt", appName, "-o", "jsonpath={.status.url}", "-n", servingNamespace)
-}
-
-func ingressAddress(gateway string, addressType string) string {
-	return noStderrShell("kubectl", "get", "svc", gateway, "-n", "istio-system",
-		"-o", fmt.Sprintf("jsonpath={.status.loadBalancer.ingress[*]['%s']}", addressType))
-}
-
-func prepareWorkDir(t *testing.T, srcDir, workDir string, preCommands []sampleapp.Command, copies []string, postCommands []sampleapp.Command) {
-	t.Log("Prepare source project")
-	err := os.RemoveAll(workDir) // this function returns nil if path not found
-	if nil == err {
-		if _, err = os.Stat(workDir); os.IsNotExist(err) {
-			err = os.MkdirAll(workDir, 0755)
-		}
-	}
-	if nil != err {
-		t.Fatalf("Failed preparing local artifacts directory: %v", err)
-	}
-
-	for _, c := range preCommands {
-		c.Run(t)
-	}
-	for _, f := range copies {
-		src := path.Join(srcDir, f)
-		dst := path.Join(workDir, f)
-		dstDir, _ := path.Split(dst)
-		noStderrShell("mkdir", "-p", dstDir)
-		if output, err := exec.Command("cp", src, dst).CombinedOutput(); err != nil {
-			t.Fatalf("Error copying: '%s' to '%s' -err: '%v'", src, dst, strings.TrimSpace(string(output)))
-		}
-	}
-	for _, c := range postCommands {
-		c.Run(t)
-	}
-}
-
-func pushDockerImage(t *testing.T, imagePath, workDir string) {
-	t.Logf("Pushing docker image to: '%s'", imagePath)
-	if output, err := exec.Command("docker", "build", "-t", imagePath, workDir).CombinedOutput(); err != nil {
-		t.Fatalf("Error building docker image: %v", strings.TrimSpace(string(output)))
-	}
-	if output, err := exec.Command("docker", "push", imagePath).CombinedOutput(); err != nil {
-		t.Fatalf("Error pushing docker image: %v", strings.TrimSpace(string(output)))
-	}
-}
-
-func deploy(t *testing.T, yamlFilePath, yamlImagePlaceholder, imagePath string) {
-	t.Log("Creating manifest")
-	// Populate manifets file with the real path to the container
-	yamlBytes, err := ioutil.ReadFile(yamlFilePath)
-	if err != nil {
-		t.Fatalf("Failed to read file %s: %v", yamlFilePath, err)
-	}
-	content := strings.Replace(string(yamlBytes), yamlImagePlaceholder, imagePath, -1)
-	if err = ioutil.WriteFile(yamlFilePath, []byte(content), 0644); err != nil {
-		t.Fatalf("Failed to write new manifest: %v", err)
-	}
-
-	t.Logf("Deploying using kubectl and using manifest file %q", yamlFilePath)
-	// Deploy using kubectl
-	if output, err := exec.Command("kubectl", "apply", "-f", yamlFilePath).CombinedOutput(); err != nil {
-		t.Fatalf("Error running kubectl: %v", strings.TrimSpace(string(output)))
-	}
-}
-
-func checkDeployment(t *testing.T, appName, expectedOutput string) {
-	t.Log("Waiting for ingress to come up")
-	gateway := "istio-ingressgateway"
-	// Wait for ingress to come up
-	ingressAddr := ""
-	serviceHost := ""
-	timeout := ingressTimeout
-	for (ingressAddr == "" || serviceHost == "") && timeout >= 0 {
-		if serviceHost == "" {
-			serviceHost = serviceHostname(appName)
-		}
-		if ingressAddr = ingressAddress(gateway, "ip"); ingressAddr == "" {
-			ingressAddr = ingressAddress(gateway, "hostname")
-		}
-		timeout -= checkInterval
-		time.Sleep(checkInterval)
-	}
-	if ingressAddr == "" || serviceHost == "" {
-		// serviceHost or ingressAddr might contain a useful error, dump them.
-		t.Fatalf("Ingress not found (ingress='%s', host='%s')", ingressAddr, serviceHost)
-	}
-	t.Logf("Curling %s/%s", ingressAddr, serviceHost)
-
-	serviceHost = strings.Replace(serviceHost, "http://", "", 1)
-	outputString := ""
-	timeout = servingTimeout
-	for outputString != expectedOutput && timeout >= 0 {
-		output, err := exec.Command("curl", "--header", "Host:"+serviceHost, "http://"+ingressAddr).Output()
-		errorString := "none"
-		if err != nil {
-			errorString = err.Error()
-		}
-		outputString = strings.TrimSpace(string(output))
-		t.Logf("App replied with '%s' (error: %s)", outputString, errorString)
-		timeout -= checkInterval
-		time.Sleep(checkInterval)
-	}
-
-	if outputString != expectedOutput {
-		t.Fatal("Timeout waiting for app to start serving")
-	}
-	t.Log("App is serving")
-}
-
-// SampleAppTestBase tests individual sample app
-func SampleAppTestBase(t *testing.T, lc sampleapp.LanguageConfig, expectedOutput string) {
-	t.Parallel()
-	imagePath := test.ImagePath(lc.AppName)
-	yamlFilePath := path.Join(lc.WorkDir, "service.yaml")
-
-	CleanupOnInterrupt(func() { cleanup(yamlFilePath, lc.WorkDir) })
-	defer cleanup(yamlFilePath, lc.WorkDir)
-
-	prepareWorkDir(t, lc.SrcDir, lc.WorkDir, lc.PreCommands, lc.Copies, lc.PostCommands)
-	pushDockerImage(t, imagePath, lc.WorkDir)
-
-	// Deploy and test
-	deploy(t, yamlFilePath, lc.YamlImagePlaceholder, imagePath)
-	checkDeployment(t, lc.AppName, expectedOutput)
-}

test/flag.go

@@ -18,8 +18,6 @@ package test
 
 import (
 	"flag"
-	"fmt"
-	"os"
 	"strings"
 )
 
@@ -29,40 +27,16 @@ var Flags = initializeFlags()
 
 // EnvironmentFlags define the flags that are needed to run the e2e tests.
 type EnvironmentFlags struct {
-	Cluster     string // K8s cluster (defaults to cluster in kubeconfig)
-	LogVerbose  bool   // Enable verbose logging
-	DockerRepo  string // Docker repo (defaults to $KO_DOCKER_REPO)
-	EmitMetrics bool   // Emit metrics
-	Tag         string // Docker image tag
 	Languages string // Allowed languages to run
 }
 
 func initializeFlags() *EnvironmentFlags {
 	var f EnvironmentFlags
-	flag.StringVar(&f.Cluster, "cluster", "",
-		"Provide the cluster to test against. Defaults to the current cluster in kubeconfig.")
-
-	flag.BoolVar(&f.LogVerbose, "logverbose", false,
-		"Set this flag to true if you would like to see verbose logging.")
-
-	flag.BoolVar(&f.EmitMetrics, "emitmetrics", false,
-		"Set this flag to true if you would like tests to emit metrics, e.g. latency of resources being realized in the system.")
-
-	flag.StringVar(&f.DockerRepo, "dockerrepo", os.Getenv("KO_DOCKER_REPO"),
-		"Provide the uri of the docker repo you have uploaded the test image to using `uploadtestimage.sh`. Defaults to $KO_DOCKER_REPO")
-
-	flag.StringVar(&f.Tag, "tag", "latest", "Provide the version tag for the test images.")
-
 	flag.StringVar(&f.Languages, "languages", "", "Comma separated languages to run e2e test on.")
 
 	return &f
 }
 
-// ImagePath is a helper function to prefix image name with repo and suffix with tag
-func ImagePath(name string) string {
-	return fmt.Sprintf("%s/%s:%s", Flags.DockerRepo, name, Flags.Tag)
-}
-
 // GetAllowedLanguages is a helper function to return a map of allowed languages based on Languages filter
 func GetAllowedLanguages() map[string]bool {
 	allowed := make(map[string]bool)

test/sampleapp/config.go

@@ -19,9 +19,6 @@ package sampleapp
 import (
 	"fmt"
 	"io/ioutil"
-	"os/exec"
-	"strings"
-	"testing"
 
 	yaml "gopkg.in/yaml.v2"
 )
@@ -29,9 +26,6 @@ import (
 const (
 	// using these defaults if not provided, see useDefaultIfNotProvided function below
 	defaultSrcDir = "../../docs/serving/samples/hello-world/helloworld-%s"
-	defaultWorkDir              = "helloworld-%s_tmp"
-	defaultAppName              = "helloworld-%s"
-	defaultYamlImagePlaceHolder = "docker.io/{username}/helloworld-%s"
 
 	// ActionMsg serves as documentation purpose, which will be referenced for
 	// clearly displaying error messages.
@@ -52,44 +46,15 @@ type AllConfigs struct {
 // LanguageConfig contains all information for building/deploying an app
 type LanguageConfig struct {
 	Language string   `yaml:"language"`
-	ExpectedOutput       string    `yaml:"expectedOutput"`
 	SrcDir   string   `yaml:"srcDir"` // Directory contains sample code
-	WorkDir              string    `yaml:"workDir"` // Temp work directory
+	Copies   []string `yaml:"copies"` // Files to be copied by the user from SrcDir
-	AppName              string    `yaml:"appName"`
-	YamlImagePlaceholder string    `yaml:"yamlImagePlaceholder"` // Token to be replaced by real docker image URI
-	PreCommands          []Command `yaml:"preCommands"`          // Commands to be ran before copying
-	Copies               []string  `yaml:"copies"`               // Files to be copied from SrcDir to WorkDir
-	PostCommands         []Command `yaml:"postCommands"`         // Commands to be ran after copying
 }
 
-// Command contains shell commands
+// UseDefaultIfNotProvided sets default value of SrcDir if not provided
-type Command struct {
-	Exec string `yaml:"exec"`
-	Args string `yaml:"args"`
-}
-
-// UseDefaultIfNotProvided sets default value to SrcDir, WorkDir, AppName, and YamlImagePlaceholder if not provided
 func (lc *LanguageConfig) UseDefaultIfNotProvided() {
 	if "" == lc.SrcDir {
 		lc.SrcDir = fmt.Sprintf(defaultSrcDir, lc.Language)
 	}
-	if "" == lc.WorkDir {
-		lc.WorkDir = fmt.Sprintf(defaultWorkDir, lc.Language)
-	}
-	if "" == lc.AppName {
-		lc.AppName = fmt.Sprintf(defaultAppName, lc.Language)
-	}
-	if "" == lc.YamlImagePlaceholder {
-		lc.YamlImagePlaceholder = fmt.Sprintf(defaultYamlImagePlaceHolder, lc.Language)
-	}
-}
-
-// Run runs command and fail if it failed
-func (c *Command) Run(t *testing.T) {
-	args := strings.Split(c.Args, " ")
-	if output, err := exec.Command(c.Exec, args...).CombinedOutput(); err != nil {
-		t.Fatalf("Error executing: '%s' '%s' -err: '%v'", c.Exec, c.Args, strings.TrimSpace(string(output)))
-	}
 }
 
 // GetConfigs parses a config yaml file and return AllConfigs struct

test/sampleapp/config.yaml

@@ -1,63 +1,43 @@
 languages:
   - language: "java-spring"
-    expectedOutput: "Hello Spring Boot Sample v1!"
-    preCommands:
-      - exec: "unzip"
-        args: "../sampleapp/helloworld-java-spring.zip -d helloworld-java-spring_tmp"
     copies:
       - "/src/main/java/com/example/helloworld/HelloworldApplication.java"
       - "service.yaml"
       - "Dockerfile"
   - language: "go"
-    expectedOutput: "Hello Go Sample v1!"
-    preCommands:
-      - exec: "cp"
-        args: "../../docs/serving/samples/hello-world/helloworld-go/go.mod helloworld-go_tmp/go.mod"
     copies:
       - "helloworld.go"
       - "service.yaml"
       - "Dockerfile"
   - language: "kotlin"
-    expectedOutput: "Hello Kotlin Sample v1!"
     copies:
       - "/src/main/kotlin/com/example/hello/Main.kt"
       - "build.gradle"
       - "service.yaml"
       - "Dockerfile"
   - language: "nodejs"
-    expectedOutput: "Hello Node.js Sample v1!"
-    preCommands:
-      - exec: "cp"
-        args: "../../docs/serving/samples/hello-world/helloworld-nodejs/package-lock.json helloworld-nodejs_tmp/package-lock.json"
     copies:
       - "index.js"
       - "package.json"
       - "service.yaml"
       - "Dockerfile"
   - language: "php"
-    expectedOutput: "Hello PHP Sample v1!"
     copies:
       - "index.php"
       - "service.yaml"
       - "Dockerfile"
   - language: "python"
-    expectedOutput: "Hello Python Sample v1!"
     copies:
       - "app.py"
       - "service.yaml"
       - "Dockerfile"
   - language: "ruby"
-    expectedOutput: "Hello Ruby Sample v1!"
-    preCommands:
-      - exec: "cp"
-        args: "../../docs/serving/samples/hello-world/helloworld-ruby/Gemfile.lock helloworld-ruby_tmp/Gemfile.lock"
     copies:
       - "app.rb"
       - "Gemfile"
       - "service.yaml"
       - "Dockerfile"
   - language: "shell"
-    expectedOutput: "Hello Shell Sample v1!"
     copies:
       - "script.sh"
       - "service.yaml"