Restructure E2E test suite (#307)

* Restructure E2E test suite

* move out more tests from the basic workflow and put them in separate
files, including tests for routes
* rename revision_workflow_test.go to revision_test.go

* Run all tests in parallel

test/e2e/basic_workflow_test.go

@@ -26,6 +26,7 @@ import (
 )
 
 func TestBasicWorkflow(t *testing.T) {
+	t.Parallel()
 	test := NewE2eTest(t)
 	test.Setup(t)
 	defer test.Teardown(t)
@@ -34,60 +35,42 @@ func TestBasicWorkflow(t *testing.T) {
 		test.serviceListEmpty(t)
 	})
 
-	t.Run("create hello service and returns no error", func(t *testing.T) {
+	t.Run("create hello service and return no error", func(t *testing.T) {
 		test.serviceCreate(t, "hello")
 	})
 
-	t.Run("create hello service again and get service already exists error", func(t *testing.T) {
+	t.Run("return valid info about hello service", func(t *testing.T) {
-		test.serviceCreateDuplicate(t, "hello")
-	})
-
-	t.Run("returns valid info about hello service", func(t *testing.T) {
 		test.serviceList(t, "hello")
 		test.serviceDescribe(t, "hello")
-		test.serviceDescribeWithPrintFlags(t, "hello")
 	})
 
-	t.Run("update hello service's configuration and returns no error", func(t *testing.T) {
+	t.Run("update hello service's configuration and return no error", func(t *testing.T) {
 		test.serviceUpdate(t, "hello", []string{"--env", "TARGET=kn", "--port", "8888"})
 	})
 
-	t.Run("create another service and returns no error", func(t *testing.T) {
+	t.Run("create another service and return no error", func(t *testing.T) {
 		test.serviceCreate(t, "svc2")
 	})
 
-	t.Run("returns a list of revisions associated with hello and svc2 services", func(t *testing.T) {
+	t.Run("return a list of revisions associated with hello and svc2 services", func(t *testing.T) {
 		test.revisionListForService(t, "hello")
 		test.revisionListForService(t, "svc2")
 	})
 
-	t.Run("returns a list of routes associated with hello and svc2 services", func(t *testing.T) {
+	t.Run("describe revision from hello service", func(t *testing.T) {
-		test.routeList(t)
+		test.revisionDescribe(t, "hello")
-		test.routeListWithArgument(t, "hello")
-		test.routeListWithPrintFlags(t, "hello", "svc2")
 	})
 
-	t.Run("describe route from hello service", func(t *testing.T) {
+	t.Run("delete hello and svc2 services and return no error", func(t *testing.T) {
-		test.routeDescribe(t, "hello")
-		test.routeDescribeWithPrintFlags(t, "hello")
-	})
-
-	t.Run("delete hello and svc2 services and returns no error", func(t *testing.T) {
 		test.serviceDelete(t, "hello")
 		test.serviceDelete(t, "svc2")
 	})
 
-	t.Run("delete hello service again and get an error", func(t *testing.T) {
+	t.Run("return no service after completing tests", func(t *testing.T) {
-		test.serviceDeleteNonexistent(t, "hello")
-	})
-
-	t.Run("returns no service after completing tests", func(t *testing.T) {
 		test.serviceListEmpty(t)
 	})
 }
 
-// Private test functions
-
 func (test *e2eTest) serviceListEmpty(t *testing.T) {
 	out, err := test.kn.RunWithOpts([]string{"service", "list"}, runOpts{NoNamespace: false})
 	assert.NilError(t, err)
@@ -110,29 +93,6 @@ func (test *e2eTest) serviceList(t *testing.T, serviceName string) {
 	assert.Check(t, util.ContainsAll(out, serviceName))
 }
 
-func (test *e2eTest) serviceCreateDuplicate(t *testing.T, serviceName string) {
-	out, err := test.kn.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false})
-	assert.Check(t, strings.Contains(out, serviceName), "The service does not exist yet")
-
-	_, err = test.kn.RunWithOpts([]string{"service", "create", serviceName,
-		"--image", KnDefaultTestImage}, runOpts{NoNamespace: false, AllowError: true})
-
-	assert.ErrorContains(t, err, "the service already exists")
-}
-
-func (test *e2eTest) revisionListForService(t *testing.T, serviceName string) {
-	out, err := test.kn.RunWithOpts([]string{"revision", "list", "-s", serviceName}, runOpts{NoNamespace: false})
-	assert.NilError(t, err)
-
-	outputLines := strings.Split(out, "\n")
-	// Ignore the last line because it is an empty string caused by splitting a line break
-	// at the end of the output string
-	for _, line := range outputLines[1 : len(outputLines)-1] {
-		// The last item is the revision status, which should be ready
-		assert.Check(t, util.ContainsAll(line, " "+serviceName+" ", "True"))
-	}
-}
-
 func (test *e2eTest) serviceDescribe(t *testing.T, serviceName string) {
 	out, err := test.kn.RunWithOpts([]string{"service", "describe", serviceName}, runOpts{NoNamespace: false})
 	assert.NilError(t, err)
@@ -155,30 +115,6 @@ func (test *e2eTest) serviceUpdate(t *testing.T, serviceName string, args []stri
 	assert.Check(t, util.ContainsAll(out, expectedOutput))
 }
 
-func (test *e2eTest) serviceDescribeWithPrintFlags(t *testing.T, serviceName string) {
-	out, err := test.kn.RunWithOpts([]string{"service", "describe", serviceName, "-o=name"}, runOpts{})
-	assert.NilError(t, err)
-
-	expectedName := fmt.Sprintf("service.serving.knative.dev/%s", serviceName)
-	assert.Equal(t, strings.TrimSpace(out), expectedName)
-}
-
-func (test *e2eTest) routeList(t *testing.T) {
-	out, err := test.kn.RunWithOpts([]string{"route", "list"}, runOpts{})
-	assert.NilError(t, err)
-
-	expectedHeaders := []string{"NAME", "URL", "AGE", "CONDITIONS", "TRAFFIC"}
-	assert.Check(t, util.ContainsAll(out, expectedHeaders...))
-}
-
-func (test *e2eTest) routeListWithArgument(t *testing.T, routeName string) {
-	out, err := test.kn.RunWithOpts([]string{"route", "list", routeName}, runOpts{})
-	assert.NilError(t, err)
-
-	expectedOutput := fmt.Sprintf("100%% -> %s", routeName)
-	assert.Check(t, util.ContainsAll(out, routeName, expectedOutput))
-}
-
 func (test *e2eTest) serviceDelete(t *testing.T, serviceName string) {
 	out, err := test.kn.RunWithOpts([]string{"service", "delete", serviceName}, runOpts{NoNamespace: false})
 	assert.NilError(t, err)
@@ -186,38 +122,28 @@ func (test *e2eTest) serviceDelete(t *testing.T, serviceName string) {
 	assert.Check(t, util.ContainsAll(out, "Service", serviceName, "successfully deleted in namespace", test.kn.namespace))
 }
 
-func (test *e2eTest) serviceDeleteNonexistent(t *testing.T, serviceName string) {
+func (test *e2eTest) revisionListForService(t *testing.T, serviceName string) {
-	out, err := test.kn.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false})
+	out, err := test.kn.RunWithOpts([]string{"revision", "list", "-s", serviceName}, runOpts{NoNamespace: false})
-	assert.Check(t, !strings.Contains(out, serviceName), "The service exists")
+	assert.NilError(t, err)
 
-	_, err = test.kn.RunWithOpts([]string{"service", "delete", serviceName}, runOpts{NoNamespace: false, AllowError: true})
+	outputLines := strings.Split(out, "\n")
-
+	// Ignore the last line because it is an empty string caused by splitting a line break
-	expectedErr := fmt.Sprintf(`services.serving.knative.dev "%s" not found`, serviceName)
+	// at the end of the output string
-	assert.ErrorContains(t, err, expectedErr)
+	for _, line := range outputLines[1 : len(outputLines)-1] {
+		// The last item is the revision status, which should be ready
+		assert.Check(t, util.ContainsAll(line, " "+serviceName+" ", "True"))
+	}
 }
 
-func (test *e2eTest) routeDescribe(t *testing.T, routeName string) {
+func (test *e2eTest) revisionDescribe(t *testing.T, serviceName string) {
-	out, err := test.kn.RunWithOpts([]string{"route", "describe", routeName}, runOpts{})
+	revName := test.findRevision(t, serviceName)
+
+	out, err := test.kn.RunWithOpts([]string{"revision", "describe", revName}, runOpts{})
 	assert.NilError(t, err)
 
 	expectedGVK := `apiVersion: serving.knative.dev/v1alpha1
-kind: Route`
+kind: Revision`
 	expectedNamespace := fmt.Sprintf("namespace: %s", test.kn.namespace)
-	expectedServiceLabel := fmt.Sprintf("serving.knative.dev/service: %s", routeName)
+	expectedServiceLabel := fmt.Sprintf("serving.knative.dev/service: %s", serviceName)
 	assert.Check(t, util.ContainsAll(out, expectedGVK, expectedNamespace, expectedServiceLabel))
 }
-
-func (test *e2eTest) routeDescribeWithPrintFlags(t *testing.T, routeName string) {
-	out, err := test.kn.RunWithOpts([]string{"route", "describe", routeName, "-o=name"}, runOpts{})
-	assert.NilError(t, err)
-
-	expectedName := fmt.Sprintf("route.serving.knative.dev/%s", routeName)
-	assert.Equal(t, strings.TrimSpace(out), expectedName)
-}
-
-func (test *e2eTest) routeListWithPrintFlags(t *testing.T, names ...string) {
-	out, err := test.kn.RunWithOpts([]string{"route", "list", "-o=jsonpath={.items[*].metadata.name}"}, runOpts{})
-	assert.NilError(t, err)
-
-	assert.Check(t, util.ContainsAll(out, names...))
-}

test/e2e/common.go

@@ -22,6 +22,7 @@ import (
 	"os/exec"
 	"regexp"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 )
@@ -42,6 +43,8 @@ const (
 	RetrySleepDuration time.Duration = 30 * time.Second
 )
 
+var m sync.Mutex
+
 type e2eTest struct {
 	env env
 	kn  kn
@@ -55,12 +58,19 @@ func NewE2eTest(t *testing.T) *e2eTest {
 
 // Setup set up an enviroment for kn integration test returns the Teardown cleanup function
 func (test *e2eTest) Setup(t *testing.T) {
-	test.env.Namespace = fmt.Sprintf("%s%d", test.env.Namespace, namespaceCount)
+	test.env.Namespace = fmt.Sprintf("%s%d", test.env.Namespace, getNamespaceCountAndIncrement())
-	namespaceCount++
 	test.kn = kn{t, test.env.Namespace, Logger{}}
 	test.CreateTestNamespace(t, test.env.Namespace)
 }
 
+func getNamespaceCountAndIncrement() int {
+	m.Lock()
+	defer m.Unlock()
+	current := namespaceCount
+	namespaceCount++
+	return current
+}
+
 // Teardown clean up
 func (test *e2eTest) Teardown(t *testing.T) {
 	test.DeleteTestNamespace(t, test.env.Namespace)

test/e2e/revision_test.go

@@ -25,25 +25,25 @@ import (
 	"gotest.tools/assert"
 )
 
-func TestRevisionWorkflow(t *testing.T) {
+func TestRevision(t *testing.T) {
+	t.Parallel()
 	test := NewE2eTest(t)
 	test.Setup(t)
 	defer test.Teardown(t)
 
-	t.Run("create hello service and returns no error", func(t *testing.T) {
+	t.Run("create hello service and return no error", func(t *testing.T) {
 		test.serviceCreate(t, "hello")
 	})
 
-	t.Run("describe revision from hello service", func(t *testing.T) {
+	t.Run("describe revision from hello service with print flags", func(t *testing.T) {
-		test.revisionDescribe(t, "hello")
 		test.revisionDescribeWithPrintFlags(t, "hello")
 	})
 
-	t.Run("delete latest revision from hello service and returns no error", func(t *testing.T) {
+	t.Run("delete latest revision from hello service and return no error", func(t *testing.T) {
 		test.revisionDelete(t, "hello")
 	})
 
-	t.Run("delete hello service and returns no error", func(t *testing.T) {
+	t.Run("delete hello service and return no error", func(t *testing.T) {
 		test.serviceDelete(t, "hello")
 	})
 }
@@ -57,19 +57,6 @@ func (test *e2eTest) revisionDelete(t *testing.T, serviceName string) {
 	assert.Check(t, util.ContainsAll(out, "Revision", revName, "deleted", "namespace", test.kn.namespace))
 }
 
-func (test *e2eTest) revisionDescribe(t *testing.T, serviceName string) {
-	revName := test.findRevision(t, serviceName)
-
-	out, err := test.kn.RunWithOpts([]string{"revision", "describe", revName}, runOpts{})
-	assert.NilError(t, err)
-
-	expectedGVK := `apiVersion: serving.knative.dev/v1alpha1
-kind: Revision`
-	expectedNamespace := fmt.Sprintf("namespace: %s", test.kn.namespace)
-	expectedServiceLabel := fmt.Sprintf("serving.knative.dev/service: %s", serviceName)
-	assert.Check(t, util.ContainsAll(out, expectedGVK, expectedNamespace, expectedServiceLabel))
-}
-
 func (test *e2eTest) revisionDescribeWithPrintFlags(t *testing.T, serviceName string) {
 	revName := test.findRevision(t, serviceName)
 

test/e2e/route_test.go

@@ -0,0 +1,103 @@
+// Copyright 2019 The Knative Authors
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build e2e
+
+package e2e
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/knative/client/pkg/util"
+	"gotest.tools/assert"
+)
+
+func TestRoute(t *testing.T) {
+	t.Parallel()
+	test := NewE2eTest(t)
+	test.Setup(t)
+	defer test.Teardown(t)
+
+	t.Run("create hello service and return no error", func(t *testing.T) {
+		test.serviceCreate(t, "hello")
+	})
+
+	t.Run("return a list of routes", func(t *testing.T) {
+		test.routeList(t)
+	})
+
+	t.Run("return a list of routes associated with hello service", func(t *testing.T) {
+		test.routeListWithArgument(t, "hello")
+	})
+
+	t.Run("return a list of routes associated with hello service with print flags", func(t *testing.T) {
+		test.routeListWithPrintFlags(t, "hello")
+	})
+
+	t.Run("describe route from hello service", func(t *testing.T) {
+		test.routeDescribe(t, "hello")
+	})
+
+	t.Run("describe route from hello service with print flags", func(t *testing.T) {
+		test.routeDescribeWithPrintFlags(t, "hello")
+	})
+
+	t.Run("delete hello service and return no error", func(t *testing.T) {
+		test.serviceDelete(t, "hello")
+	})
+}
+
+func (test *e2eTest) routeList(t *testing.T) {
+	out, err := test.kn.RunWithOpts([]string{"route", "list"}, runOpts{})
+	assert.NilError(t, err)
+
+	expectedHeaders := []string{"NAME", "URL", "AGE", "CONDITIONS", "TRAFFIC"}
+	assert.Check(t, util.ContainsAll(out, expectedHeaders...))
+}
+
+func (test *e2eTest) routeListWithArgument(t *testing.T, routeName string) {
+	out, err := test.kn.RunWithOpts([]string{"route", "list", routeName}, runOpts{})
+	assert.NilError(t, err)
+
+	expectedOutput := fmt.Sprintf("100%% -> %s", routeName)
+	assert.Check(t, util.ContainsAll(out, routeName, expectedOutput))
+}
+
+func (test *e2eTest) routeDescribe(t *testing.T, routeName string) {
+	out, err := test.kn.RunWithOpts([]string{"route", "describe", routeName}, runOpts{})
+	assert.NilError(t, err)
+
+	expectedGVK := `apiVersion: serving.knative.dev/v1alpha1
+kind: Route`
+	expectedNamespace := fmt.Sprintf("namespace: %s", test.kn.namespace)
+	expectedServiceLabel := fmt.Sprintf("serving.knative.dev/service: %s", routeName)
+	assert.Check(t, util.ContainsAll(out, expectedGVK, expectedNamespace, expectedServiceLabel))
+}
+
+func (test *e2eTest) routeDescribeWithPrintFlags(t *testing.T, routeName string) {
+	out, err := test.kn.RunWithOpts([]string{"route", "describe", routeName, "-o=name"}, runOpts{})
+	assert.NilError(t, err)
+
+	expectedName := fmt.Sprintf("route.serving.knative.dev/%s", routeName)
+	assert.Equal(t, strings.TrimSpace(out), expectedName)
+}
+
+func (test *e2eTest) routeListWithPrintFlags(t *testing.T, names ...string) {
+	out, err := test.kn.RunWithOpts([]string{"route", "list", "-o=jsonpath={.items[*].metadata.name}"}, runOpts{})
+	assert.NilError(t, err)
+
+	assert.Check(t, util.ContainsAll(out, names...))
+}

test/e2e/service_options_test.go

@@ -24,6 +24,7 @@ import (
 )
 
 func TestServiceOptions(t *testing.T) {
+	t.Parallel()
 	test := NewE2eTest(t)
 	test.Setup(t)
 	defer test.Teardown(t)

test/e2e/service_test.go

@@ -0,0 +1,74 @@
+// Copyright 2019 The Knative Authors
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build e2e
+
+package e2e
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"gotest.tools/assert"
+)
+
+func TestService(t *testing.T) {
+	t.Parallel()
+	test := NewE2eTest(t)
+	test.Setup(t)
+	defer test.Teardown(t)
+
+	t.Run("create hello service duplicate and get service already exists error", func(t *testing.T) {
+		test.serviceCreate(t, "hello")
+		test.serviceCreateDuplicate(t, "hello")
+	})
+
+	t.Run("return valid info about hello service with print flags", func(t *testing.T) {
+		test.serviceDescribeWithPrintFlags(t, "hello")
+	})
+
+	t.Run("delete hello service repeatedly and get an error", func(t *testing.T) {
+		test.serviceDelete(t, "hello")
+		test.serviceDeleteNonexistent(t, "hello")
+	})
+}
+
+func (test *e2eTest) serviceCreateDuplicate(t *testing.T, serviceName string) {
+	out, err := test.kn.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false})
+	assert.Check(t, strings.Contains(out, serviceName), "The service does not exist yet")
+
+	_, err = test.kn.RunWithOpts([]string{"service", "create", serviceName,
+		"--image", KnDefaultTestImage}, runOpts{NoNamespace: false, AllowError: true})
+
+	assert.ErrorContains(t, err, "the service already exists")
+}
+
+func (test *e2eTest) serviceDescribeWithPrintFlags(t *testing.T, serviceName string) {
+	out, err := test.kn.RunWithOpts([]string{"service", "describe", serviceName, "-o=name"}, runOpts{})
+	assert.NilError(t, err)
+
+	expectedName := fmt.Sprintf("service.serving.knative.dev/%s", serviceName)
+	assert.Equal(t, strings.TrimSpace(out), expectedName)
+}
+
+func (test *e2eTest) serviceDeleteNonexistent(t *testing.T, serviceName string) {
+	out, err := test.kn.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false})
+	assert.Check(t, !strings.Contains(out, serviceName), "The service exists")
+
+	_, err = test.kn.RunWithOpts([]string{"service", "delete", serviceName}, runOpts{NoNamespace: false, AllowError: true})
+
+	expectedErr := fmt.Sprintf(`services.serving.knative.dev "%s" not found`, serviceName)
+	assert.ErrorContains(t, err, expectedErr)
+}

test/e2e/version_test.go

@@ -24,6 +24,7 @@ import (
 )
 
 func TestVersion(t *testing.T) {
+	t.Parallel()
 	env := buildEnv(t)
 	kn := kn{t, env.Namespace, Logger{}}