Test framework enhancement for multicluster doc test (#8160)

* test framework enhancement for multicluster doc test

* simplify multicluster test framework with setup=multicluster directive

* restore documentation comment

content/en/docs/tasks/observability/distributed-tracing/zipkin/multicluster_test.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090,SC2154,SC2155,SC2034
+
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -u
+set -o pipefail
+
+# @setup multicluster
+
+# TODO: Remove this once we have a real multicluster test
+
+echo "start multicluster test with KUBECONFIG=${KUBECONFIG}"
+echo "DOCTEST_KUBECONFIG=${DOCTEST_KUBECONFIG}"
+echo "DOCTEST_NETWORK_TOPOLOGY=${DOCTEST_NETWORK_TOPOLOGY}"
+
+IFS=',' read -r -a CONFIGS <<< "${KUBECONFIG}"
+for kcfg in "${CONFIGS[@]}"; do
+  kubectl --kubeconfig="$kcfg" get pods -A
+done
+
+# @cleanup
+set +e
+echo "end multicluster test"

pkg/test/istioio/framework.go

@@ -145,6 +145,7 @@ func checkFile(path string) (*TestCase, error) {
 	// find setup configuration
 	re = regexp.MustCompile(fmt.Sprintf("(?m)^%v (.*)$", setupSpec))
 	setups := re.FindAllStringSubmatch(testScript, -1)
+
 	if numSetups := len(setups); numSetups != 1 {
 		err := fmt.Errorf(
 			"script error: expected one line that starts with '%v', got %v line(s)",

pkg/test/istioio/script.go

@@ -142,7 +142,11 @@ func (s Script) getEnv(ctx Context, fileName string) []string {
 	}
 	customVars[testDebugFile] = fileName + "_debug.txt"
 
-	customVars[kubeConfigEnvVar] = ctx.KubeEnv().Settings().KubeConfig[0]
+	if ctx.TestContext.Clusters().IsMulticluster() {
+		customVars[kubeConfigEnvVar] = strings.Join(ctx.KubeEnv().Settings().KubeConfig, ",")
+	} else {
+		customVars[kubeConfigEnvVar] = ctx.KubeEnv().Settings().KubeConfig[0]
+	}
 
 	for k, v := range s.Env {
 		customVars[k] = v

prow/config/topology/multi-cluster.json

@@ -1,6 +1,6 @@
 [
   {
-      "cluster_name": "doc_cluster1",
+      "cluster_name": "doc-cluster1",
       "pod_subnet": "10.10.0.0/16",
       "svc_subnet": "10.255.10.0/24",
       "network_id": 0,
@@ -8,7 +8,7 @@
       "config_index": 0
   },
   {
-      "cluster_name": "doc_cluster2",
+      "cluster_name": "doc-cluster2",
       "pod_subnet": "10.20.0.0/16",
       "svc_subnet": "10.255.20.0/24",
       "network_id": 0,
@@ -16,7 +16,7 @@
       "config_index": 0
   },
   {
-     "cluster_name": "doc_cluster3",
+     "cluster_name": "doc-cluster3",
       "pod_subnet": "10.30.0.0/16",
       "svc_subnet": "10.255.30.0/24",
       "network_id": 1,

prow/integ-suite-kind.sh

@@ -101,7 +101,17 @@ if [[ -z "${SKIP_SETUP:-}" ]]; then
 
     export TEST_ENV=kind-metallb
     export DOCTEST_KUBECONFIG
-    DOCTEST_KUBECONFIG=$(IFS=':'; echo "${KUBECONFIGS[*]}")
+    DOCTEST_KUBECONFIG=$(IFS=','; echo "${KUBECONFIGS[*]}")
+
+    ITER_END=$((NUM_CLUSTERS-1))
+    declare -a NETWORK_TOPOLOGIES
+
+    for i in $(seq 0 $ITER_END); do
+      NETWORK_TOPOLOGIES+=("$i:test-network-${CLUSTER_NETWORK_ID[$i]}")
+    done
+
+    export DOCTEST_NETWORK_TOPOLOGY
+    DOCTEST_NETWORK_TOPOLOGY=$(IFS=','; echo "${NETWORK_TOPOLOGIES[*]}")
   fi
 fi
 

tests/setup/multicluster/doc_test.go

@@ -0,0 +1,40 @@
+// Copyright 2020 Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package setupconfig
+
+import (
+	"os"
+	"testing"
+
+	"istio.io/istio.io/pkg/test/istioio"
+	"istio.io/istio/pkg/test/framework"
+)
+
+var (
+	setupSpec = "multicluster"
+)
+
+func TestMain(m *testing.M) {
+	if !istioio.NeedSetup(setupSpec) {
+		os.Exit(0)
+	}
+	framework.
+		NewSuite(m).
+		RequireMinClusters(2).
+		Run()
+}
+
+func TestDocs(t *testing.T) {
+	istioio.TestDocs(t, setupSpec)
+}

tests/setup/profile_default/doc_test.go

@@ -35,6 +35,7 @@ func TestMain(m *testing.M) {
 	framework.
 		NewSuite(m).
 		Setup(istio.Setup(&inst, nil)).
+		RequireSingleCluster().
 		Run()
 }
 

tests/setup/profile_demo/doc_test.go

@@ -36,6 +36,7 @@ func TestMain(m *testing.M) {
 	framework.
 		NewSuite(m).
 		Setup(istio.Setup(&inst, setupConfig)).
+		RequireSingleCluster().
 		Run()
 }
 

tests/setup/profile_none/doc_test.go

@@ -29,7 +29,9 @@ func TestMain(m *testing.M) {
 	if !istioio.NeedSetup(setupSpec) {
 		os.Exit(0)
 	}
-	framework.NewSuite(m).Run()
+	framework.NewSuite(m).
+		RequireSingleCluster().
+		Run()
 }
 
 func TestDocs(t *testing.T) {

tests/tests.mk

@@ -1,22 +1,35 @@
 DEFAULT_TIMEOUT=60m
 export TIMEOUT ?= ${DEFAULT_TIMEOUT}
 
+_DOCTEST_FLAGS ?= ${DOCTEST_FLAGS}
+_DOCTEST_KUBECONFIG ?= $(DOCTEST_KUBECONFIG)
+ifneq ($(_DOCTEST_KUBECONFIG),)
+	_DOCTEST_FLAGS += --istio.test.kube.config=$(_DOCTEST_KUBECONFIG)
+endif
+
+_DOCTEST_NETWORK_TOPOLOGY ?= $(DOCTEST_NETWORK_TOPOLOGY)
+ifneq ($(_DOCTEST_NETWORK_TOPOLOGY),)
+	_DOCTEST_FLAGS += --istio.test.kube.networkTopology=$(_DOCTEST_NETWORK_TOPOLOGY)
+endif
+
 # gocache disabled by -count=1
 # tests in different packages forced to be sequential by -p=1
 doc.test.%: init | $(JUNIT_REPORT)
-	@${GO} test ${REPO_ROOT}/tests/setup/$*/... \
+	${GO} test ${REPO_ROOT}/tests/setup/$*/... \
 		-v -timeout=${TIMEOUT} -count=1 -p=1 \
 		-istio.test.hub=$(HUB) \
 		-istio.test.tag=$(TAG) \
+		${_DOCTEST_FLAGS} \
 		2>&1 | tee >($(JUNIT_REPORT) > $(JUNIT_OUT))
 
 # gocache disabled by -count=1
 # tests in different packages forced to be sequential by -p=1
 doc.test: init | $(JUNIT_REPORT)
-	@${GO} test ${REPO_ROOT}/tests/setup/... \
+	${GO} test ${REPO_ROOT}/tests/setup/... \
 		-v -timeout=${TIMEOUT} -count=1 -p=1 \
 		-istio.test.hub=$(HUB) \
 		-istio.test.tag=$(TAG) \
+		${_DOCTEST_FLAGS} \
 		2>&1 | tee >($(JUNIT_REPORT) > $(JUNIT_OUT))
 
 doc.test.help: