Multicluster integration test (#4998)

This implements the run_multicluster_test() function in bin/_test-helpers.sh.

The idea is to create two clusters (source and target) using k3d, with linkerd and multicluster support in both, plus emojivoto (without vote-bot) in target, and vote-bot in source.
We then link the clusters and make sure traffic is flowing.

Detailed sequence:

Create certficates.
Install linkerd along with multicluster support in the target cluster.
Run the target1 test: install emojivoto in the target cluster (without vote-bot).
Run linkerd mc link on the target cluster.
Install linkerd along with multicluster support in the source cluster.
Apply the link resource in the source cluster.
Run the source test: Check linkerd mc gateways returns the target cluster link, and only install emojivoto's vote-bot in the source cluster. Note vote-bot's yaml defines the web-svc service as web-svc-target.emojivoto:80
Run the target2 test: Make sure web-svc in the target cluster is receiving requests.

bin/_test-helpers.sh

@@ -223,6 +223,11 @@ start_k3d_test() {
   fi
 }
 
+multicluster_link() {
+  lbIP=$(kubectl --context="$context" get svc -n kube-system traefik -o 'go-template={{ (index .status.loadBalancer.ingress 0).ip }}')
+  "$linkerd_path" multicluster link --api-server-address "https://${lbIP}:6443" --cluster-name "$1"
+}
+
 get_test_config() {
   local name=$1
   config=''
@@ -375,10 +380,23 @@ run_uninstall_test() {
 }
 
 run_multicluster_test() {
-  export context="k3d-source"
-  run_test "$test_directory/install_test.go" --multicluster
+  tmp=$(mktemp -d -t l5dcerts.XXX)
+  pwd=$PWD
+  cd "$tmp"
+  "$bindir"/certs-openssl
+  cd "$pwd"
   export context="k3d-target"
-  run_test "$test_directory/install_test.go" --multicluster
+  run_test "$test_directory/install_test.go" --multicluster --certs-path "$tmp"
+  run_test "$test_directory/multicluster/target1" --multicluster
+  link=$(multicluster_link target)
+
+  export context="k3d-source"
+  run_test "$test_directory/install_test.go" --multicluster --certs-path "$tmp"
+  echo "$link" | kubectl --context="$context" apply -f -
+  run_test "$test_directory/multicluster/source" --multicluster
+
+  export context="k3d-target"
+  run_test "$test_directory/multicluster/target2" --multicluster
 }
 
 run_deep_test() {

test/integration/multicluster/source/source_test.go

@@ -0,0 +1,70 @@
+package source
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/linkerd/linkerd2/testutil"
+)
+
+var TestHelper *testutil.TestHelper
+
+func TestMain(m *testing.M) {
+	TestHelper = testutil.NewTestHelper()
+	if !TestHelper.Multicluster() {
+		fmt.Fprintln(os.Stderr, "Multicluster test disabled")
+		os.Exit(0)
+	}
+	os.Exit(testutil.Run(m, TestHelper))
+}
+
+func TestGateways(t *testing.T) {
+	timeout := time.Minute
+	err := TestHelper.RetryFor(timeout, func() error {
+		out, stderr, err := TestHelper.LinkerdRun("multicluster", "gateways")
+		if err != nil {
+			return fmt.Errorf("%s\n%s", err, stderr)
+		}
+		rows := strings.Split(out, "\n")
+		if len(rows) < 2 {
+			return errors.New("response is empty")
+		}
+		fields := strings.Fields(rows[1])
+		if len(fields) < 6 {
+			return fmt.Errorf("unexpected number of columns: %d", len(fields))
+		}
+		if fields[0] != "target" {
+			return fmt.Errorf("unexpected target cluster name: %s", fields[0])
+		}
+		if fields[1] != "True" {
+			return errors.New("target cluster is not alive")
+		}
+		if fields[2] != "1" {
+			return fmt.Errorf("invalid NUM_SVC: %s", fields[2])
+		}
+
+		return nil
+	})
+	if err != nil {
+		testutil.AnnotatedFatal(t, fmt.Sprintf("'linkerd multicluster gateways' command timed-out (%s)", timeout), err)
+	}
+}
+
+func TestInstallVoteBot(t *testing.T) {
+	if err := TestHelper.CreateDataPlaneNamespaceIfNotExists("emojivoto", nil); err != nil {
+		testutil.AnnotatedFatalf(t, "failed to create emojivoto namespace",
+			"failed to create emojivoto namespace: %s", err)
+	}
+	yaml, err := testutil.ReadFile("testdata/vote-bot.yml")
+	if err != nil {
+		testutil.AnnotatedFatalf(t, "failed to read 'vote_bot.yml'", "failed to read 'vote_bot.yml': %s", err)
+	}
+	o, err := TestHelper.KubectlApply(yaml, "emojivoto")
+	if err != nil {
+		testutil.AnnotatedFatalf(t, "failed to install vote-bot", "failed to install vote-bot: %s\n%s", err, o)
+	}
+}

test/integration/multicluster/source/testdata/vote-bot.yml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: vote-bot
+    app.kubernetes.io/part-of: emojivoto
+    app.kubernetes.io/version: v10
+  name: vote-bot
+  namespace: emojivoto
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: vote-bot
+      version: v10
+  template:
+    metadata:
+      annotations:
+        linkerd.io/inject: enabled
+      labels:
+        app: vote-bot
+        version: v10
+    spec:
+      containers:
+      - command:
+        - emojivoto-vote-bot
+        env:
+        - name: WEB_HOST
+          value: web-svc-target.emojivoto:80
+        image: buoyantio/emojivoto-web:v10
+        name: vote-bot
+        resources:
+          requests:
+            cpu: 10m
+---

test/integration/multicluster/target1/target_test.go

@@ -0,0 +1,35 @@
+package target1
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/linkerd/linkerd2/testutil"
+)
+
+var TestHelper *testutil.TestHelper
+
+func TestMain(m *testing.M) {
+	TestHelper = testutil.NewTestHelper()
+	if !TestHelper.Multicluster() {
+		fmt.Fprintln(os.Stderr, "Multicluster test disabled")
+		os.Exit(0)
+	}
+	os.Exit(testutil.Run(m, TestHelper))
+}
+
+func TestInstallEmojivoto(t *testing.T) {
+	if err := TestHelper.CreateDataPlaneNamespaceIfNotExists("emojivoto", nil); err != nil {
+		testutil.AnnotatedFatalf(t, "failed to create emojivoto namespace",
+			"failed to create emojivoto namespace: %s", err)
+	}
+	yaml, err := testutil.ReadFile("testdata/emojivoto-no-bot.yml")
+	if err != nil {
+		testutil.AnnotatedFatalf(t, "failed to read 'emojivoto-no-bot.yml'", "failed to read 'emojivoto-no-bot.yml': %s", err)
+	}
+	out, err := TestHelper.KubectlApply(yaml, "emojivoto")
+	if err != nil {
+		testutil.AnnotatedFatalf(t, "failed to install emojivoto", "failed to install emojivoto: %s\n%s", err, out)
+	}
+}

test/integration/multicluster/target1/testdata/emojivoto-no-bot.yml

@@ -0,0 +1,190 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: emoji
+  namespace: emojivoto
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: voting
+  namespace: emojivoto
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: web
+  namespace: emojivoto
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: emoji-svc
+  namespace: emojivoto
+spec:
+  ports:
+  - name: grpc
+    port: 8080
+    targetPort: 8080
+  - name: prom
+    port: 8801
+    targetPort: 8801
+  selector:
+    app: emoji-svc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: voting-svc
+  namespace: emojivoto
+spec:
+  ports:
+  - name: grpc
+    port: 8080
+    targetPort: 8080
+  - name: prom
+    port: 8801
+    targetPort: 8801
+  selector:
+    app: voting-svc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: web-svc
+  namespace: emojivoto
+  labels:
+    mirror.linkerd.io/exported: "true"
+spec:
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8080
+  selector:
+    app: web-svc
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: emoji
+    app.kubernetes.io/part-of: emojivoto
+    app.kubernetes.io/version: v10
+  name: emoji
+  namespace: emojivoto
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: emoji-svc
+      version: v10
+  template:
+    metadata:
+      annotations:
+        linkerd.io/inject: enabled
+      labels:
+        app: emoji-svc
+        version: v10
+    spec:
+      containers:
+      - env:
+        - name: GRPC_PORT
+          value: "8080"
+        - name: PROM_PORT
+          value: "8801"
+        image: buoyantio/emojivoto-emoji-svc:v10
+        name: emoji-svc
+        ports:
+        - containerPort: 8080
+          name: grpc
+        - containerPort: 8801
+          name: prom
+        resources:
+          requests:
+            cpu: 100m
+      serviceAccountName: emoji
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: voting
+    app.kubernetes.io/part-of: emojivoto
+    app.kubernetes.io/version: v10
+  name: voting
+  namespace: emojivoto
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: voting-svc
+      version: v10
+  template:
+    metadata:
+      annotations:
+        linkerd.io/inject: enabled
+      labels:
+        app: voting-svc
+        version: v10
+    spec:
+      containers:
+      - env:
+        - name: GRPC_PORT
+          value: "8080"
+        - name: PROM_PORT
+          value: "8801"
+        image: buoyantio/emojivoto-voting-svc:v10
+        name: voting-svc
+        ports:
+        - containerPort: 8080
+          name: grpc
+        - containerPort: 8801
+          name: prom
+        resources:
+          requests:
+            cpu: 100m
+      serviceAccountName: voting
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: web
+    app.kubernetes.io/part-of: emojivoto
+    app.kubernetes.io/version: v10
+  name: web
+  namespace: emojivoto
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: web-svc
+      version: v10
+  template:
+    metadata:
+      annotations:
+        linkerd.io/inject: enabled
+      labels:
+        app: web-svc
+        version: v10
+    spec:
+      containers:
+      - env:
+        - name: WEB_PORT
+          value: "8080"
+        - name: EMOJISVC_HOST
+          value: emoji-svc.emojivoto:8080
+        - name: VOTINGSVC_HOST
+          value: voting-svc.emojivoto:8080
+        - name: INDEX_BUNDLE
+          value: dist/index_bundle.js
+        image: buoyantio/emojivoto-web:v10
+        name: web-svc
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: 100m
+      serviceAccountName: web

test/integration/multicluster/target2/target_test.go

@@ -0,0 +1,47 @@
+package target2
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/linkerd/linkerd2/testutil"
+)
+
+var TestHelper *testutil.TestHelper
+
+func TestMain(m *testing.M) {
+	TestHelper = testutil.NewTestHelper()
+	if !TestHelper.Multicluster() {
+		fmt.Fprintln(os.Stderr, "Multicluster test disabled")
+		os.Exit(0)
+	}
+	os.Exit(testutil.Run(m, TestHelper))
+}
+
+func TestTargetTraffic(t *testing.T) {
+	timeout := time.Minute
+	err := TestHelper.RetryFor(timeout, func() error {
+		out, err := TestHelper.Kubectl("",
+			"--namespace", "emojivoto",
+			"logs",
+			"--selector", "app=web-svc",
+			"--container", "web-svc",
+		)
+		if err != nil {
+			return fmt.Errorf("%s\n%s", err, out)
+		}
+		for _, row := range strings.Split(out, "\n") {
+			if strings.Contains(row, "http://web-svc.emojivoto.svc.cluster.local:80/api/vote") {
+				return nil
+			}
+		}
+		return errors.New("web-svc logs in target cluster were empty")
+	})
+	if err != nil {
+		testutil.AnnotatedFatal(t, fmt.Sprintf("'linkerd multicluster gateways' command timed-out (%s)", timeout), err)
+	}
+}