From b98e45e721f3107561d4c65be2ef285951cebfe4 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Mon, 29 Jul 2019 15:00:05 +0200 Subject: [PATCH] Restructure E2E test suite (#307) * Restructure E2E test suite * move out more tests from the basic workflow and put them in separate files, including tests for routes * rename revision_workflow_test.go to revision_test.go * Run all tests in parallel --- test/e2e/basic_workflow_test.go | 126 ++++-------------- test/e2e/common.go | 14 +- ...sion_workflow_test.go => revision_test.go} | 25 +--- test/e2e/route_test.go | 103 ++++++++++++++ test/e2e/service_options_test.go | 1 + test/e2e/service_test.go | 74 ++++++++++ test/e2e/version_test.go | 1 + 7 files changed, 223 insertions(+), 121 deletions(-) rename test/e2e/{revision_workflow_test.go => revision_test.go} (68%) create mode 100644 test/e2e/route_test.go create mode 100644 test/e2e/service_test.go diff --git a/test/e2e/basic_workflow_test.go b/test/e2e/basic_workflow_test.go index 446217179..a4337a7f5 100644 --- a/test/e2e/basic_workflow_test.go +++ b/test/e2e/basic_workflow_test.go @@ -26,6 +26,7 @@ import ( ) func TestBasicWorkflow(t *testing.T) { + t.Parallel() test := NewE2eTest(t) test.Setup(t) defer test.Teardown(t) @@ -34,60 +35,42 @@ func TestBasicWorkflow(t *testing.T) { test.serviceListEmpty(t) }) - t.Run("create hello service and returns no error", func(t *testing.T) { + t.Run("create hello service and return no error", func(t *testing.T) { test.serviceCreate(t, "hello") }) - t.Run("create hello service again and get service already exists error", func(t *testing.T) { - test.serviceCreateDuplicate(t, "hello") - }) - - t.Run("returns valid info about hello service", func(t *testing.T) { + t.Run("return valid info about hello service", func(t *testing.T) { test.serviceList(t, "hello") test.serviceDescribe(t, "hello") - test.serviceDescribeWithPrintFlags(t, "hello") }) - t.Run("update hello service's configuration and returns no error", func(t *testing.T) { + t.Run("update hello service's configuration and return no error", func(t *testing.T) { test.serviceUpdate(t, "hello", []string{"--env", "TARGET=kn", "--port", "8888"}) }) - t.Run("create another service and returns no error", func(t *testing.T) { + t.Run("create another service and return no error", func(t *testing.T) { test.serviceCreate(t, "svc2") }) - t.Run("returns a list of revisions associated with hello and svc2 services", func(t *testing.T) { + t.Run("return a list of revisions associated with hello and svc2 services", func(t *testing.T) { test.revisionListForService(t, "hello") test.revisionListForService(t, "svc2") }) - t.Run("returns a list of routes associated with hello and svc2 services", func(t *testing.T) { - test.routeList(t) - test.routeListWithArgument(t, "hello") - test.routeListWithPrintFlags(t, "hello", "svc2") + t.Run("describe revision from hello service", func(t *testing.T) { + test.revisionDescribe(t, "hello") }) - t.Run("describe route from hello service", func(t *testing.T) { - test.routeDescribe(t, "hello") - test.routeDescribeWithPrintFlags(t, "hello") - }) - - t.Run("delete hello and svc2 services and returns no error", func(t *testing.T) { + t.Run("delete hello and svc2 services and return no error", func(t *testing.T) { test.serviceDelete(t, "hello") test.serviceDelete(t, "svc2") }) - t.Run("delete hello service again and get an error", func(t *testing.T) { - test.serviceDeleteNonexistent(t, "hello") - }) - - t.Run("returns no service after completing tests", func(t *testing.T) { + t.Run("return no service after completing tests", func(t *testing.T) { test.serviceListEmpty(t) }) } -// Private test functions - func (test *e2eTest) serviceListEmpty(t *testing.T) { out, err := test.kn.RunWithOpts([]string{"service", "list"}, runOpts{NoNamespace: false}) assert.NilError(t, err) @@ -110,29 +93,6 @@ func (test *e2eTest) serviceList(t *testing.T, serviceName string) { assert.Check(t, util.ContainsAll(out, serviceName)) } -func (test *e2eTest) serviceCreateDuplicate(t *testing.T, serviceName string) { - out, err := test.kn.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false}) - assert.Check(t, strings.Contains(out, serviceName), "The service does not exist yet") - - _, err = test.kn.RunWithOpts([]string{"service", "create", serviceName, - "--image", KnDefaultTestImage}, runOpts{NoNamespace: false, AllowError: true}) - - assert.ErrorContains(t, err, "the service already exists") -} - -func (test *e2eTest) revisionListForService(t *testing.T, serviceName string) { - out, err := test.kn.RunWithOpts([]string{"revision", "list", "-s", serviceName}, runOpts{NoNamespace: false}) - assert.NilError(t, err) - - outputLines := strings.Split(out, "\n") - // Ignore the last line because it is an empty string caused by splitting a line break - // at the end of the output string - for _, line := range outputLines[1 : len(outputLines)-1] { - // The last item is the revision status, which should be ready - assert.Check(t, util.ContainsAll(line, " "+serviceName+" ", "True")) - } -} - func (test *e2eTest) serviceDescribe(t *testing.T, serviceName string) { out, err := test.kn.RunWithOpts([]string{"service", "describe", serviceName}, runOpts{NoNamespace: false}) assert.NilError(t, err) @@ -155,30 +115,6 @@ func (test *e2eTest) serviceUpdate(t *testing.T, serviceName string, args []stri assert.Check(t, util.ContainsAll(out, expectedOutput)) } -func (test *e2eTest) serviceDescribeWithPrintFlags(t *testing.T, serviceName string) { - out, err := test.kn.RunWithOpts([]string{"service", "describe", serviceName, "-o=name"}, runOpts{}) - assert.NilError(t, err) - - expectedName := fmt.Sprintf("service.serving.knative.dev/%s", serviceName) - assert.Equal(t, strings.TrimSpace(out), expectedName) -} - -func (test *e2eTest) routeList(t *testing.T) { - out, err := test.kn.RunWithOpts([]string{"route", "list"}, runOpts{}) - assert.NilError(t, err) - - expectedHeaders := []string{"NAME", "URL", "AGE", "CONDITIONS", "TRAFFIC"} - assert.Check(t, util.ContainsAll(out, expectedHeaders...)) -} - -func (test *e2eTest) routeListWithArgument(t *testing.T, routeName string) { - out, err := test.kn.RunWithOpts([]string{"route", "list", routeName}, runOpts{}) - assert.NilError(t, err) - - expectedOutput := fmt.Sprintf("100%% -> %s", routeName) - assert.Check(t, util.ContainsAll(out, routeName, expectedOutput)) -} - func (test *e2eTest) serviceDelete(t *testing.T, serviceName string) { out, err := test.kn.RunWithOpts([]string{"service", "delete", serviceName}, runOpts{NoNamespace: false}) assert.NilError(t, err) @@ -186,38 +122,28 @@ func (test *e2eTest) serviceDelete(t *testing.T, serviceName string) { assert.Check(t, util.ContainsAll(out, "Service", serviceName, "successfully deleted in namespace", test.kn.namespace)) } -func (test *e2eTest) serviceDeleteNonexistent(t *testing.T, serviceName string) { - out, err := test.kn.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false}) - assert.Check(t, !strings.Contains(out, serviceName), "The service exists") +func (test *e2eTest) revisionListForService(t *testing.T, serviceName string) { + out, err := test.kn.RunWithOpts([]string{"revision", "list", "-s", serviceName}, runOpts{NoNamespace: false}) + assert.NilError(t, err) - _, err = test.kn.RunWithOpts([]string{"service", "delete", serviceName}, runOpts{NoNamespace: false, AllowError: true}) - - expectedErr := fmt.Sprintf(`services.serving.knative.dev "%s" not found`, serviceName) - assert.ErrorContains(t, err, expectedErr) + outputLines := strings.Split(out, "\n") + // Ignore the last line because it is an empty string caused by splitting a line break + // at the end of the output string + for _, line := range outputLines[1 : len(outputLines)-1] { + // The last item is the revision status, which should be ready + assert.Check(t, util.ContainsAll(line, " "+serviceName+" ", "True")) + } } -func (test *e2eTest) routeDescribe(t *testing.T, routeName string) { - out, err := test.kn.RunWithOpts([]string{"route", "describe", routeName}, runOpts{}) +func (test *e2eTest) revisionDescribe(t *testing.T, serviceName string) { + revName := test.findRevision(t, serviceName) + + out, err := test.kn.RunWithOpts([]string{"revision", "describe", revName}, runOpts{}) assert.NilError(t, err) expectedGVK := `apiVersion: serving.knative.dev/v1alpha1 -kind: Route` +kind: Revision` expectedNamespace := fmt.Sprintf("namespace: %s", test.kn.namespace) - expectedServiceLabel := fmt.Sprintf("serving.knative.dev/service: %s", routeName) + expectedServiceLabel := fmt.Sprintf("serving.knative.dev/service: %s", serviceName) assert.Check(t, util.ContainsAll(out, expectedGVK, expectedNamespace, expectedServiceLabel)) } - -func (test *e2eTest) routeDescribeWithPrintFlags(t *testing.T, routeName string) { - out, err := test.kn.RunWithOpts([]string{"route", "describe", routeName, "-o=name"}, runOpts{}) - assert.NilError(t, err) - - expectedName := fmt.Sprintf("route.serving.knative.dev/%s", routeName) - assert.Equal(t, strings.TrimSpace(out), expectedName) -} - -func (test *e2eTest) routeListWithPrintFlags(t *testing.T, names ...string) { - out, err := test.kn.RunWithOpts([]string{"route", "list", "-o=jsonpath={.items[*].metadata.name}"}, runOpts{}) - assert.NilError(t, err) - - assert.Check(t, util.ContainsAll(out, names...)) -} diff --git a/test/e2e/common.go b/test/e2e/common.go index 09f2dcc68..cebfd372a 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -22,6 +22,7 @@ import ( "os/exec" "regexp" "strings" + "sync" "testing" "time" ) @@ -42,6 +43,8 @@ const ( RetrySleepDuration time.Duration = 30 * time.Second ) +var m sync.Mutex + type e2eTest struct { env env kn kn @@ -55,12 +58,19 @@ func NewE2eTest(t *testing.T) *e2eTest { // Setup set up an enviroment for kn integration test returns the Teardown cleanup function func (test *e2eTest) Setup(t *testing.T) { - test.env.Namespace = fmt.Sprintf("%s%d", test.env.Namespace, namespaceCount) - namespaceCount++ + test.env.Namespace = fmt.Sprintf("%s%d", test.env.Namespace, getNamespaceCountAndIncrement()) test.kn = kn{t, test.env.Namespace, Logger{}} test.CreateTestNamespace(t, test.env.Namespace) } +func getNamespaceCountAndIncrement() int { + m.Lock() + defer m.Unlock() + current := namespaceCount + namespaceCount++ + return current +} + // Teardown clean up func (test *e2eTest) Teardown(t *testing.T) { test.DeleteTestNamespace(t, test.env.Namespace) diff --git a/test/e2e/revision_workflow_test.go b/test/e2e/revision_test.go similarity index 68% rename from test/e2e/revision_workflow_test.go rename to test/e2e/revision_test.go index add5c5f31..d58b1a065 100644 --- a/test/e2e/revision_workflow_test.go +++ b/test/e2e/revision_test.go @@ -25,25 +25,25 @@ import ( "gotest.tools/assert" ) -func TestRevisionWorkflow(t *testing.T) { +func TestRevision(t *testing.T) { + t.Parallel() test := NewE2eTest(t) test.Setup(t) defer test.Teardown(t) - t.Run("create hello service and returns no error", func(t *testing.T) { + t.Run("create hello service and return no error", func(t *testing.T) { test.serviceCreate(t, "hello") }) - t.Run("describe revision from hello service", func(t *testing.T) { - test.revisionDescribe(t, "hello") + t.Run("describe revision from hello service with print flags", func(t *testing.T) { test.revisionDescribeWithPrintFlags(t, "hello") }) - t.Run("delete latest revision from hello service and returns no error", func(t *testing.T) { + t.Run("delete latest revision from hello service and return no error", func(t *testing.T) { test.revisionDelete(t, "hello") }) - t.Run("delete hello service and returns no error", func(t *testing.T) { + t.Run("delete hello service and return no error", func(t *testing.T) { test.serviceDelete(t, "hello") }) } @@ -57,19 +57,6 @@ func (test *e2eTest) revisionDelete(t *testing.T, serviceName string) { assert.Check(t, util.ContainsAll(out, "Revision", revName, "deleted", "namespace", test.kn.namespace)) } -func (test *e2eTest) revisionDescribe(t *testing.T, serviceName string) { - revName := test.findRevision(t, serviceName) - - out, err := test.kn.RunWithOpts([]string{"revision", "describe", revName}, runOpts{}) - assert.NilError(t, err) - - expectedGVK := `apiVersion: serving.knative.dev/v1alpha1 -kind: Revision` - expectedNamespace := fmt.Sprintf("namespace: %s", test.kn.namespace) - expectedServiceLabel := fmt.Sprintf("serving.knative.dev/service: %s", serviceName) - assert.Check(t, util.ContainsAll(out, expectedGVK, expectedNamespace, expectedServiceLabel)) -} - func (test *e2eTest) revisionDescribeWithPrintFlags(t *testing.T, serviceName string) { revName := test.findRevision(t, serviceName) diff --git a/test/e2e/route_test.go b/test/e2e/route_test.go new file mode 100644 index 000000000..f2a94535d --- /dev/null +++ b/test/e2e/route_test.go @@ -0,0 +1,103 @@ +// Copyright 2019 The Knative Authors + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build e2e + +package e2e + +import ( + "fmt" + "strings" + "testing" + + "github.com/knative/client/pkg/util" + "gotest.tools/assert" +) + +func TestRoute(t *testing.T) { + t.Parallel() + test := NewE2eTest(t) + test.Setup(t) + defer test.Teardown(t) + + t.Run("create hello service and return no error", func(t *testing.T) { + test.serviceCreate(t, "hello") + }) + + t.Run("return a list of routes", func(t *testing.T) { + test.routeList(t) + }) + + t.Run("return a list of routes associated with hello service", func(t *testing.T) { + test.routeListWithArgument(t, "hello") + }) + + t.Run("return a list of routes associated with hello service with print flags", func(t *testing.T) { + test.routeListWithPrintFlags(t, "hello") + }) + + t.Run("describe route from hello service", func(t *testing.T) { + test.routeDescribe(t, "hello") + }) + + t.Run("describe route from hello service with print flags", func(t *testing.T) { + test.routeDescribeWithPrintFlags(t, "hello") + }) + + t.Run("delete hello service and return no error", func(t *testing.T) { + test.serviceDelete(t, "hello") + }) +} + +func (test *e2eTest) routeList(t *testing.T) { + out, err := test.kn.RunWithOpts([]string{"route", "list"}, runOpts{}) + assert.NilError(t, err) + + expectedHeaders := []string{"NAME", "URL", "AGE", "CONDITIONS", "TRAFFIC"} + assert.Check(t, util.ContainsAll(out, expectedHeaders...)) +} + +func (test *e2eTest) routeListWithArgument(t *testing.T, routeName string) { + out, err := test.kn.RunWithOpts([]string{"route", "list", routeName}, runOpts{}) + assert.NilError(t, err) + + expectedOutput := fmt.Sprintf("100%% -> %s", routeName) + assert.Check(t, util.ContainsAll(out, routeName, expectedOutput)) +} + +func (test *e2eTest) routeDescribe(t *testing.T, routeName string) { + out, err := test.kn.RunWithOpts([]string{"route", "describe", routeName}, runOpts{}) + assert.NilError(t, err) + + expectedGVK := `apiVersion: serving.knative.dev/v1alpha1 +kind: Route` + expectedNamespace := fmt.Sprintf("namespace: %s", test.kn.namespace) + expectedServiceLabel := fmt.Sprintf("serving.knative.dev/service: %s", routeName) + assert.Check(t, util.ContainsAll(out, expectedGVK, expectedNamespace, expectedServiceLabel)) +} + +func (test *e2eTest) routeDescribeWithPrintFlags(t *testing.T, routeName string) { + out, err := test.kn.RunWithOpts([]string{"route", "describe", routeName, "-o=name"}, runOpts{}) + assert.NilError(t, err) + + expectedName := fmt.Sprintf("route.serving.knative.dev/%s", routeName) + assert.Equal(t, strings.TrimSpace(out), expectedName) +} + +func (test *e2eTest) routeListWithPrintFlags(t *testing.T, names ...string) { + out, err := test.kn.RunWithOpts([]string{"route", "list", "-o=jsonpath={.items[*].metadata.name}"}, runOpts{}) + assert.NilError(t, err) + + assert.Check(t, util.ContainsAll(out, names...)) +} diff --git a/test/e2e/service_options_test.go b/test/e2e/service_options_test.go index ddf7838b6..a55a1b425 100644 --- a/test/e2e/service_options_test.go +++ b/test/e2e/service_options_test.go @@ -24,6 +24,7 @@ import ( ) func TestServiceOptions(t *testing.T) { + t.Parallel() test := NewE2eTest(t) test.Setup(t) defer test.Teardown(t) diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go new file mode 100644 index 000000000..f8662af8c --- /dev/null +++ b/test/e2e/service_test.go @@ -0,0 +1,74 @@ +// Copyright 2019 The Knative Authors + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build e2e + +package e2e + +import ( + "fmt" + "strings" + "testing" + + "gotest.tools/assert" +) + +func TestService(t *testing.T) { + t.Parallel() + test := NewE2eTest(t) + test.Setup(t) + defer test.Teardown(t) + + t.Run("create hello service duplicate and get service already exists error", func(t *testing.T) { + test.serviceCreate(t, "hello") + test.serviceCreateDuplicate(t, "hello") + }) + + t.Run("return valid info about hello service with print flags", func(t *testing.T) { + test.serviceDescribeWithPrintFlags(t, "hello") + }) + + t.Run("delete hello service repeatedly and get an error", func(t *testing.T) { + test.serviceDelete(t, "hello") + test.serviceDeleteNonexistent(t, "hello") + }) +} + +func (test *e2eTest) serviceCreateDuplicate(t *testing.T, serviceName string) { + out, err := test.kn.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false}) + assert.Check(t, strings.Contains(out, serviceName), "The service does not exist yet") + + _, err = test.kn.RunWithOpts([]string{"service", "create", serviceName, + "--image", KnDefaultTestImage}, runOpts{NoNamespace: false, AllowError: true}) + + assert.ErrorContains(t, err, "the service already exists") +} + +func (test *e2eTest) serviceDescribeWithPrintFlags(t *testing.T, serviceName string) { + out, err := test.kn.RunWithOpts([]string{"service", "describe", serviceName, "-o=name"}, runOpts{}) + assert.NilError(t, err) + + expectedName := fmt.Sprintf("service.serving.knative.dev/%s", serviceName) + assert.Equal(t, strings.TrimSpace(out), expectedName) +} + +func (test *e2eTest) serviceDeleteNonexistent(t *testing.T, serviceName string) { + out, err := test.kn.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false}) + assert.Check(t, !strings.Contains(out, serviceName), "The service exists") + + _, err = test.kn.RunWithOpts([]string{"service", "delete", serviceName}, runOpts{NoNamespace: false, AllowError: true}) + + expectedErr := fmt.Sprintf(`services.serving.knative.dev "%s" not found`, serviceName) + assert.ErrorContains(t, err, expectedErr) +} diff --git a/test/e2e/version_test.go b/test/e2e/version_test.go index 45aebfca9..a8bb7f780 100644 --- a/test/e2e/version_test.go +++ b/test/e2e/version_test.go @@ -24,6 +24,7 @@ import ( ) func TestVersion(t *testing.T) { + t.Parallel() env := buildEnv(t) kn := kn{t, env.Namespace, Logger{}}