Integration tests for Tekton (#528)

* Integration tests for Tekton

* Use knative/client instead of fork

* also put back the e2e-tests into its original format where it doesn't
run the tekton tests

* Run ./hack/build.sh with Go 1.12

* Pull buildah and kn tasks directly from catalog

* Revert "Use knative/client instead of fork"

This reverts commit 2ab272f587.

* Revert "Revert "Use knative/client instead of fork""

This reverts commit f14c2105e2.

* Update Tekton to 0.8.0

* Revert "Use knative/client instead of fork"

This reverts commit 2ab272f587.

* Use knative/client instead of fork

This reverts commit 8eb87ada02.

* Export variables after calling initialize

* Run against arbitrary Docker registry

* Conditionally install Tekton

* Use knative/client instead of fork

This reverts commit 3800adbf69.

* Revert "Use knative/client instead of fork"

This reverts commit 71a3d33b5a.

* Simplify passing the flag to test

* Simplify imports

test/e2e-common.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/e2e-tests.sh
+
+function cluster_setup() {
+  header "Building client"
+  ${REPO_ROOT_DIR}/hack/build.sh -f || return 1
+}
+
+function knative_setup() {
+  local version=${KNATIVE_VERSION:-latest}
+  header "Installing Knative serving (${version})"
+
+  if [ "${version}" = "latest" ]; then
+    start_latest_knative_serving
+  else
+    start_release_knative_serving "${version}"
+  fi
+}

test/e2e-smoke-tests.sh

@@ -25,15 +25,7 @@
 # the cluster.
 
 source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/e2e-tests.sh
-
-# Helper functions.
-
-# Build kn before integration tests, so we fail fast in case of error.
-./hack/build.sh -f
-
-function knative_setup() {
-  start_latest_knative_serving
-}
+source $(dirname $0)/e2e-common.sh
 
 # Will create and delete this namespace and use it for smoke tests
 export KN_E2E_SMOKE_TESTS_NAMESPACE=kne2esmoketests

test/e2e-tests.sh

@@ -29,29 +29,10 @@
 # of this specified version will be installed in the Kubernetes cluster, and
 # all the tests will run against Knative serving of this specific version.
 source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/e2e-tests.sh
-
-# Helper functions.
-
-# Build kn before integration tests, so we fail fast in case of error.
-function cluster_setup() {
-  header "Building client"
-  ${REPO_ROOT_DIR}/hack/build.sh -u || return 1
-}
-
-function knative_setup() {
-  local version=${KNATIVE_VERSION:-latest}
-  header "Installing Knative serving (${version})"
-
-  if [ "${version}" = "latest" ]; then
-    start_latest_knative_serving
-  else
-    start_release_knative_serving "${version}"
-  fi
-}
+source $(dirname $0)/e2e-common.sh
 
 # Add local dir to have access to built kn
 export PATH=$PATH:${REPO_ROOT_DIR}
-export KNATIVE_VERSION=${KNATIVE_VERSION:-latest}
 
 # Script entry point.
 

test/e2e/common.go

@@ -220,3 +220,11 @@ func matchRegexp(t *testing.T, matchingRegexp, actual string) bool {
 	}
 	return matched
 }
+
+func currentDir(t *testing.T) string {
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatal("Unable to read current dir:", err)
+	}
+	return dir
+}

test/e2e/e2e_flags.go

@@ -18,13 +18,28 @@ package e2e
 
 import (
 	"flag"
+	"os"
 )
 
-var _ = initializeFlags()
+// Flags holds the command line flags or defaults for settings in the user's environment.
+// See ClientFlags for the list of supported fields.
+var Flags = initializeFlags()
 
-func initializeFlags() bool {
-	// emitmetrics is a required flag for running periodic test jobs, add it here as a no-op to avoid the error
-	emitMetrics := flag.Bool("emitmetrics", false,
-		"Set this flag to true if you would like tests to emit metrics, e.g. latency of resources being realized in the system.")
-	return *emitMetrics
+// ClientFlags define the flags that are needed to run the e2e tests.
+type ClientFlags struct {
+	EmitMetrics      bool
+	DockerConfigJSON string
+}
+
+func initializeFlags() *ClientFlags {
+	var f ClientFlags
+	// emitmetrics is a required flag for running periodic test jobs, add it here as a no-op to avoid the error
+	flag.BoolVar(&f.EmitMetrics, "emitmetrics", false,
+		"Set this flag to true if you would like tests to emit metrics, e.g. latency of resources being realized in the system.")
+
+	dockerConfigJSON := os.Getenv("DOCKER_CONFIG_JSON")
+	flag.StringVar(&f.DockerConfigJSON, "dockerconfigjson", dockerConfigJSON,
+		"Provide the path to Docker configuration file in json format. Defaults to $DOCKER_CONFIG_JSON")
+
+	return &f
 }

test/e2e/tekton_test.go

@@ -0,0 +1,86 @@
+// Copyright 2019 The Knative Authors
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build tekton
+
+package e2e
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"gotest.tools/assert"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"knative.dev/client/pkg/util"
+)
+
+const (
+	// Interval specifies the time between two polls.
+	Interval = 10 * time.Second
+	// Timeout specifies the timeout for the function PollImmediate to reach a certain status.
+	Timeout = 5 * time.Minute
+)
+
+func TestTektonPipeline(t *testing.T) {
+	test := NewE2eTest(t)
+	test.Setup(t)
+
+	kubectl := kubectl{t, Logger{}}
+	basedir := currentDir(t) + "/../resources/tekton"
+
+	// create secret for the kn-deployer-account service account
+	_, err := kubectl.RunWithOpts([]string{"create", "-n", test.env.Namespace, "secret",
+		"generic", "container-registry",
+		"--from-file=.dockerconfigjson=" + Flags.DockerConfigJSON,
+		"--type=kubernetes.io/dockerconfigjson"}, runOpts{})
+	assert.NilError(t, err)
+
+	_, err = kubectl.RunWithOpts([]string{"apply", "-n", test.env.Namespace, "-f", basedir + "/kn-deployer-rbac.yaml"}, runOpts{})
+	assert.NilError(t, err)
+
+	_, err = kubectl.RunWithOpts([]string{"apply", "-n", test.env.Namespace, "-f", "https://raw.githubusercontent.com/tektoncd/catalog/master/buildah/buildah.yaml"}, runOpts{})
+	assert.NilError(t, err)
+
+	_, err = kubectl.RunWithOpts([]string{"apply", "-n", test.env.Namespace, "-f", "https://raw.githubusercontent.com/tektoncd/catalog/master/kn/kn.yaml"}, runOpts{})
+	assert.NilError(t, err)
+
+	_, err = kubectl.RunWithOpts([]string{"apply", "-n", test.env.Namespace, "-f", basedir + "/kn-pipeline.yaml"}, runOpts{})
+	assert.NilError(t, err)
+
+	_, err = kubectl.RunWithOpts([]string{"apply", "-n", test.env.Namespace, "-f", basedir + "/kn-pipeline-resource.yaml"}, runOpts{})
+	assert.NilError(t, err)
+
+	_, err = kubectl.RunWithOpts([]string{"create", "-n", test.env.Namespace, "-f", basedir + "/kn-pipeline-run.yaml"}, runOpts{})
+	assert.NilError(t, err)
+
+	err = waitForPipelineSuccess(t, kubectl, test.env.Namespace)
+	assert.NilError(t, err)
+
+	const serviceName = "hello"
+	out, err := test.kn.RunWithOpts([]string{"service", "describe", serviceName}, runOpts{NoNamespace: false})
+	assert.NilError(t, err)
+	assert.Assert(t, util.ContainsAll(out, serviceName, test.kn.namespace))
+	assert.Assert(t, util.ContainsAll(out, "Conditions", "ConfigurationsReady", "Ready", "RoutesReady"))
+
+	// tear down only if the test passes, we want to keep the pods otherwise
+	test.Teardown(t)
+}
+
+func waitForPipelineSuccess(t *testing.T, k kubectl, namespace string) error {
+	return wait.PollImmediate(Interval, Timeout, func() (bool, error) {
+		out, err := k.RunWithOpts([]string{"get", "pipelinerun", "-n", namespace, "-o=jsonpath='{.items[0].status.conditions[?(@.type==\"Succeeded\")].status}'"}, runOpts{})
+		return strings.Contains(out, "True"), err
+	})
+}

test/resources/tekton/kn-deployer-rbac-template.yaml

@@ -0,0 +1,50 @@
+# Copyright 2019 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Define a ServiceAccount named kn-deployer-account that has permission to
+# manage Knative services.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kn-deployer-account
+  namespace: ${KN_E2E_NAMESPACE}0
+secrets:
+  - name: container-registry
+imagePullSecrets:
+  - name: container-registry
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kn-deployer
+rules:
+  - apiGroups: ["serving.knative.dev"]
+    resources: ["services"]
+    verbs: ["get", "list", "create", "update", "delete", "patch", "watch"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: kn-deployer-binding
+subjects:
+- kind: ServiceAccount
+  name: kn-deployer-account
+  namespace: ${KN_E2E_NAMESPACE}0
+roleRef:
+  kind: ClusterRole
+  name: kn-deployer
+  apiGroup: rbac.authorization.k8s.io

test/resources/tekton/kn-pipeline-resource-template.yaml

@@ -0,0 +1,32 @@
+# Copyright 2019 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: tekton.dev/v1alpha1
+kind: PipelineResource
+metadata:
+  name: buildah-build-kn-create-source
+spec:
+  type: git
+  params:
+    - name: url
+      value: "https://github.com/knative/client"
+---
+apiVersion: tekton.dev/v1alpha1
+kind: PipelineResource
+metadata:
+  name: buildah-build-kn-create-image
+spec:
+  type: image
+  params:
+    - name: url
+      value: "${CONTAINER_REGISTRY}/helloworld:tkn"

test/resources/tekton/kn-pipeline-run.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: tekton.dev/v1alpha1
+kind: PipelineRun
+metadata:
+  generateName: buildah-build-kn-create-
+spec:
+  serviceAccount: kn-deployer-account
+  pipelineRef:
+    name: buildah-build-kn-create
+  resources:
+    - name: source
+      resourceRef:
+        name: buildah-build-kn-create-source
+    - name: image
+      resourceRef:
+        name: buildah-build-kn-create-image
+  params:
+    - name: ARGS
+      value:
+        - "service"
+        - "create"
+        - "hello"
+        - "--force"
+        - "--service-account=kn-deployer-account"
+        - "--image=$(inputs.resources.image.url)"
+        - "--env=TARGET=Tekton"

test/resources/tekton/kn-pipeline.yaml

@@ -0,0 +1,60 @@
+# Copyright 2019 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: tekton.dev/v1alpha1
+kind: Pipeline
+metadata:
+  name: buildah-build-kn-create
+spec:
+  resources:
+  - name: source
+    type: git
+  - name: image
+    type: image
+  params:
+  - name: ARGS
+    type: array
+    description: Arguments to pass to kn CLI
+    default:
+      - "help"
+  tasks:
+  - name: buildah-build
+    taskRef:
+      name: buildah
+    resources:
+      inputs:
+        - name: source
+          resource: source
+      outputs:
+        - name: image
+          resource: image
+    params:
+    - name: DOCKERFILE
+      value: ./test/test_images/helloworld/Dockerfile
+  - name: kn-service-create
+    taskRef:
+      name: kn
+    runAfter:
+      - buildah-build
+    resources:
+      inputs:
+      - name: image
+        resource: image
+        from:
+          - buildah-build
+    params:
+    - name: kn-image
+      value: "gcr.io/knative-nightly/knative.dev/client/cmd/kn"
+    - name: ARGS
+      value:
+        - "$(params.ARGS)"

test/tekton-tests.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script runs the integration tests with Tekton
+
+# In order to run the tests locally against a standalone Kubernetes cluster
+# and container registry, you must set the $CONTAINER_REGISTRY and $DOCKER_CONFIG_JSON
+# environment variables and login to your registry of choice.
+
+source $(dirname $0)/e2e-common.sh
+
+# Add local dir to have access to built kn
+export PATH=$PATH:${REPO_ROOT_DIR}
+
+# Script entry point.
+initialize $@
+
+export TEKTON_VERSION=${TEKTON_VERSION:-v0.8.0}
+export KN_E2E_NAMESPACE=tkn-kn
+
+header "Running integration tests for Tekton"
+
+# Install Tekton if not already installed
+if [[ $(kubectl api-resources | grep -c tekton.dev) -eq 0 ]]; then
+  kubectl apply -f https://github.com/tektoncd/pipeline/releases/download/${TEKTON_VERSION}/release.yaml
+fi
+
+if (( IS_PROW )); then
+  # Configure Docker so that we can create a secret for GCR
+  gcloud auth configure-docker
+  gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io
+  export CONTAINER_REGISTRY=gcr.io/${E2E_PROJECT_ID}/${E2E_BASE_NAME}-e2e-img/${RANDOM}
+  export DOCKER_CONFIG_JSON=/root/.docker/config.json
+fi
+
+# Feed $KN_E2E_NAMESPACE and $CONTAINER_REGISTRY into yaml files
+resource_dir=$(dirname $0)/resources/tekton
+for file in kn-deployer-rbac kn-pipeline-resource; do
+  sed -e "s#\${KN_E2E_NAMESPACE}#${KN_E2E_NAMESPACE}#" \
+      -e "s#\${CONTAINER_REGISTRY}#${CONTAINER_REGISTRY}#" ${resource_dir}/${file}-template.yaml > ${resource_dir}/${file}.yaml
+done
+
+go_test_e2e -timeout=30m -tags=tekton ./test/e2e || fail_test
+
+success

test/test_images/helloworld/Dockerfile

@@ -0,0 +1,23 @@
+# Use the offical Golang image to create a build artifact.
+# This is based on Debian and sets the GOPATH to /go.
+# https://hub.docker.com/_/golang
+FROM golang:1.12 as builder
+
+# Copy local code to the container image.
+WORKDIR /go/src/knative.dev/client/test/test_images/helloworld
+COPY ./test/test_images/helloworld/helloworld.go .
+
+# Build the command inside the container.
+# (You may fetch or manage dependencies here,
+# either manually or with a tool like "godep".)
+RUN CGO_ENABLED=0 GOOS=linux go build -v -o helloworld
+
+# Use a Docker multi-stage build to create a lean production image.
+# https://docs.docker.com/develop/develop-images/multistage-build/#use-multi-stage-builds
+FROM scratch
+
+# Copy the binary to the production image from the builder stage.
+COPY --from=builder /go/src/knative.dev/client/test/test_images/helloworld/helloworld /helloworld
+
+# Run the web service on container startup.
+CMD ["/helloworld"]

test/test_images/helloworld/helloworld.go

@@ -0,0 +1,44 @@
+// Copyright 2019 The Knative Authors
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+)
+
+func handler(w http.ResponseWriter, r *http.Request) {
+	log.Print("Hello world received a request.")
+	target := os.Getenv("TARGET")
+	if target == "" {
+		target = "World"
+	}
+	fmt.Fprintf(w, "Hello %s!\n", target)
+}
+
+func main() {
+	log.Print("Hello world sample started.")
+
+	http.HandleFunc("/", handler)
+
+	port := os.Getenv("PORT")
+	if port == "" {
+		port = "8080"
+	}
+
+	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))
+}

vendor/modules.txt

@@ -424,6 +424,7 @@ k8s.io/apimachinery/pkg/util/net
 k8s.io/apimachinery/pkg/util/yaml
 k8s.io/apimachinery/pkg/runtime/serializer
 k8s.io/apimachinery/pkg/runtime/serializer/streaming
+k8s.io/apimachinery/pkg/util/wait
 k8s.io/apimachinery/third_party/forked/golang/reflect
 k8s.io/apimachinery/pkg/apis/meta/v1/unstructured
 k8s.io/apimachinery/pkg/util/mergepatch
@@ -439,7 +440,6 @@ k8s.io/apimachinery/pkg/util/clock
 k8s.io/apimachinery/pkg/util/framer
 k8s.io/apimachinery/pkg/util/cache
 k8s.io/apimachinery/pkg/util/diff
-k8s.io/apimachinery/pkg/util/wait
 k8s.io/apimachinery/pkg/apis/meta/internalversion
 # k8s.io/cli-runtime v0.0.0-20191016113937-7693ce2cae74
 k8s.io/cli-runtime/pkg/genericclioptions