Fixes for multicluster tests (#8309)

content/en/docs/setup/install/multicluster/common.sh

@@ -36,11 +36,44 @@ function set_multi_network_vars
   export CTX_CLUSTER2="${KUBE_CONTEXTS[2]}"
 }
 
-# cleanup removes all resources created by the tests.
+# configure_trust creates a hierarchy of
-function cleanup()
+function configure_trust
 {
-  # Remove generated yaml files.
+  # Keeps the certs under a separate directory.
-  rm -f "cluster1.yaml" "cluster2.yaml"
+  mkdir -p certs
+  pushd certs || exit
+
+  # Create the root cert.
+  make -f ../tools/certs/Makefile.selfsigned.mk root-ca
+
+  # Create and deploy intermediate certs for cluster1 and cluster2.
+  make -f ../tools/certs/Makefile.selfsigned.mk cluster1-cacerts
+  make -f ../tools/certs/Makefile.selfsigned.mk cluster2-cacerts
+
+  # Create the istio-system namespace in each cluster so that we can create the secrets.
+  kubectl --context="$CTX_CLUSTER1" create namespace istio-system
+  kubectl --context="$CTX_CLUSTER2" create namespace istio-system
+
+  # Deploy secret to each cluster
+  kubectl --context="$CTX_CLUSTER1" create secret generic cacerts -n istio-system \
+      --from-file=cluster1/ca-cert.pem \
+      --from-file=cluster1/ca-key.pem \
+      --from-file=cluster1/root-cert.pem \
+      --from-file=cluster1/cert-chain.pem
+  kubectl --context="$CTX_CLUSTER2" create secret generic cacerts -n istio-system \
+      --from-file=cluster2/ca-cert.pem \
+      --from-file=cluster2/ca-key.pem \
+      --from-file=cluster2/root-cert.pem \
+      --from-file=cluster2/cert-chain.pem
+
+  popd || exit # Return to the previous directory.
+}
+
+# cleanup removes all resources created by the tests.
+function cleanup
+{
+  # Remove temp files.
+  rm -f cluster1.yaml cluster2.yaml certs
 
   # Delete the namespaces on both clusters concurrently
   delete_namespaces_cluster1 &
@@ -50,47 +83,62 @@ function cleanup()
 
 # _delete_namespaces_cluster1 removes the istio-system and sample namespaces on both
 # CLUSTER1.
-function delete_namespaces_cluster1()
+function delete_namespaces_cluster1
 {
   kubectl delete ns istio-system sample --context="${CTX_CLUSTER1}" --ignore-not-found
 }
 
 # _delete_namespaces_cluster2 removes the istio-system and sample namespaces on both
 # CLUSTER2.
-function delete_namespaces_cluster2()
+function delete_namespaces_cluster2
 {
   kubectl delete ns istio-system sample --context="${CTX_CLUSTER2}" --ignore-not-found
 }
 
 # verify_load_balancing verifies that traffic is load balanced properly
 # between CLUSTER1 and CLUSTER2.
-function verify_load_balancing()
+function verify_load_balancing
 {
-  # Deploy HelloWorld and Sleep.
+  # Deploy the HelloWorld service.
   snip_deploy_the_helloworld_service_1
   snip_deploy_the_helloworld_service_2
   snip_deploy_the_helloworld_service_3
+
+  # Deploy HelloWorld v1 and v2
   snip_deploy_helloworld_v1_1
   snip_deploy_helloworld_v2_1
+
+  # Deploy Sleep
   snip_deploy_sleep_1
-  snip_deploy_sleep_3
 
-  # Wait for the deployments in CLUSTER1
+  # Wait for all the deployments.
-  (KUBECONFIG="${KUBECONFIG_CLUSTER1}"; _wait_for_deployment sample helloworld-v1)
+  _wait_for_deployment sample helloworld-v1 "${CTX_CLUSTER1}"
-  (KUBECONFIG="${KUBECONFIG_CLUSTER1}"; _wait_for_deployment sample sleep)
+  _wait_for_deployment sample sleep "${CTX_CLUSTER1}"
+  _wait_for_deployment sample helloworld-v2 "${CTX_CLUSTER2}"
+  _wait_for_deployment sample sleep "${CTX_CLUSTER2}"
 
-  # Wait for the deployments in CLUSTER2
+  # Verify everything is deployed as expected.
-  (KUBECONFIG="${KUBECONFIG_CLUSTER2}"; _wait_for_deployment sample helloworld-v2)
+  VERIFY_RETRIES=0 # Don't retry.
-  (KUBECONFIG="${KUBECONFIG_CLUSTER2}"; _wait_for_deployment sample sleep)
+  echo "Verifying helloworld v1 deployment"
+  _verify_like snip_deploy_helloworld_v1_2 "$snip_deploy_helloworld_v1_2_out"
+  echo "Verifying helloworld v2 deployment"
+  _verify_like snip_deploy_helloworld_v2_2 "$snip_deploy_helloworld_v2_2_out"
+  echo "Verifying sleep deployment in ${CTX_CLUSTER1}"
+  _verify_like snip_deploy_sleep_2 "$snip_deploy_sleep_2_out"
+  echo "Verifying sleep deployment in ${CTX_CLUSTER2}"
+  _verify_like snip_deploy_sleep_3 "$snip_deploy_sleep_3_out"
+  unset VERIFY_RETRIES # Restore default
 
   local EXPECTED_RESPONSE_FROM_CLUSTER1="Hello version: v1, instance:"
   local EXPECTED_RESPONSE_FROM_CLUSTER2="Hello version: v2, instance:"
 
   # Verify we hit both clusters from CLUSTER1
+  echo "Verifying load balancing from ${CTX_CLUSTER1}"
   _verify_contains snip_verifying_crosscluster_traffic_1 "$EXPECTED_RESPONSE_FROM_CLUSTER1"
   _verify_contains snip_verifying_crosscluster_traffic_1 "$EXPECTED_RESPONSE_FROM_CLUSTER2"
 
   # Verify we hit both clusters from CLUSTER2
+  echo "Verifying load balancing from ${CTX_CLUSTER2}"
   _verify_contains snip_verifying_crosscluster_traffic_3 "$EXPECTED_RESPONSE_FROM_CLUSTER1"
   _verify_contains snip_verifying_crosscluster_traffic_3 "$EXPECTED_RESPONSE_FROM_CLUSTER2"
 }

content/en/docs/setup/install/multicluster/index.md

@@ -112,7 +112,7 @@ Service workloads communicate directly (pod-to-pod) across cluster boundaries.
 Create the Istio configuration for `cluster1`:
 
 {{< text bash >}}
-$ cat <<EOF > ./cluster1.yaml
+$ cat <<EOF > cluster1.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -136,7 +136,7 @@ $ istioctl install --context="${CTX_CLUSTER1}" -f cluster1.yaml
 Create the Istio configuration for `cluster2`:
 
 {{< text bash >}}
-$ cat <<EOF > ./cluster2.yaml
+$ cat <<EOF > cluster2.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -200,7 +200,7 @@ traffic. The gateway in each cluster must be reachable from the other cluster.
 Create the Istio configuration for `cluster1`:
 
 {{< text bash >}}
-$ cat <<EOF > ./cluster1.yaml
+$ cat <<EOF > cluster1.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -249,7 +249,7 @@ $ kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \
 Create the Istio configuration for `cluster2`:
 
 {{< text bash >}}
-$ cat <<EOF > ./cluster2.yaml
+$ cat <<EOF > cluster2.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -333,7 +333,7 @@ traffic.
 Create the Istio configuration for `cluster1`:
 
 {{< text bash >}}
-$ cat <<EOF > ./cluster1.yaml
+$ cat <<EOF > cluster1.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -374,6 +374,28 @@ $ kubectl apply --context="${CTX_CLUSTER1}" -f \
     @samples/multicluster/expose-istiod.yaml@
 {{< /text >}}
 
+<h3>Enable API Server Access to cluster2</h3>
+
+Before we can configure the remote cluster, we first have to give the control
+plane in `cluster1` access to the API Server in `cluster2`. This will do the
+following:
+
+- Enables the control plane to authenticate connection requests from
+  workloads running in `cluster2`. Without API Server access, the control
+  plane will reject the requests.
+
+- Enables discovery of service endpoints running in `cluster2`.
+
+To provide API Server access to `cluster2`, we generate a remote secret and
+apply it to `cluster1`:
+
+{{< text bash >}}
+$ istioctl x create-remote-secret \
+    --context="${CTX_CLUSTER2}" \
+    --name=cluster2 | \
+    kubectl apply -f - --context="${CTX_CLUSTER1}"
+{{< /text >}}
+
 <h3>Configure cluster2 as a remote</h3>
 
 Save the address of `cluster1`’s ingress gateway.
@@ -388,7 +410,7 @@ $ export DISCOVERY_ADDRESS=$(kubectl \
 Now create a remote configuration for `cluster2`.
 
 {{< text bash >}}
-$ cat <<EOF > ./cluster2.yaml
+$ cat <<EOF > cluster2.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -408,18 +430,6 @@ Apply the configuration to `cluster2`:
 $ istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml
 {{< /text >}}
 
-<h3>Enable Endpoint Discovery for cluster2</h3>
-
-Create a remote secret that will allow the control plane in `cluster1` to access the
-API Server in `cluster2` for endpoints.
-
-{{< text bash >}}
-$ istioctl x create-remote-secret \
-    --context="${CTX_CLUSTER2}" \
-    --name=cluster2 | \
-    kubectl apply -f - --context="${CTX_CLUSTER1}"
-{{< /text >}}
-
 {{< /tab >}}
 
 {{< tab name="Primary-Remote, Multi-Network" category-value="primary-remote-multi-network" >}}
@@ -452,7 +462,7 @@ same east-west gateway.
 Create the Istio configuration for `cluster1`:
 
 {{< text bash >}}
-$ cat <<EOF > ./cluster1.yaml
+$ cat <<EOF > cluster1.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -503,6 +513,28 @@ $ kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \
     @samples/multicluster/expose-services.yaml@
 {{< /text >}}
 
+<h3>Enable API Server Access to cluster2</h3>
+
+Before we can configure the remote cluster, we first have to give the control
+plane in `cluster1` access to the API Server in `cluster2`. This will do the
+following:
+
+- Enables the control plane to authenticate connection requests from
+  workloads running in `cluster2`. Without API Server access, the control
+  plane will reject the requests.
+
+- Enables discovery of service endpoints running in `cluster2`.
+
+To provide API Server access to `cluster2`, we generate a remote secret and
+apply it to `cluster1`:
+
+{{< text bash >}}
+$ istioctl x create-remote-secret \
+    --context="${CTX_CLUSTER2}" \
+    --name=cluster2 | \
+    kubectl apply -f - --context="${CTX_CLUSTER1}"
+{{< /text >}}
+
 <h3>Configure cluster2 as a remote with services exposed</h3>
 
 Save the address of `cluster1`’s ingress gateway.
@@ -517,7 +549,7 @@ $ export DISCOVERY_ADDRESS=$(kubectl \
 Now create a remote configuration on `cluster2`.
 
 {{< text bash >}}
-$ cat <<EOF > ./cluster2.yaml
+$ cat <<EOF > cluster2.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -552,18 +584,6 @@ $ kubectl --context="${CTX_CLUSTER2}" apply -n istio-system -f \
     @samples/multicluster/expose-services.yaml@
 {{< /text >}}
 
-<h3>Enable Endpoint Discovery for cluster2 on network2</h3>
-
-Create a remote secret that will allow the control plane in `cluster1` to
-access the API Server in `cluster2` for endpoints.
-
-{{< text bash >}}
-$ istioctl x create-remote-secret \
-    --context="${CTX_CLUSTER2}" \
-    --name=cluster2 | \
-    kubectl apply -f - --context="${CTX_CLUSTER1}"
-{{< /text >}}
-
 {{< /tab >}}
 
 {{< /tabset >}}
@@ -608,10 +628,10 @@ Create the `HelloWorld` service in both clusters:
 {{< text bash >}}
 $ kubectl apply --context="${CTX_CLUSTER1}" \
     -f @samples/helloworld/helloworld.yaml@ \
-    -l app=helloworld -n sample
+    -l service=helloworld -n sample
 $ kubectl apply --context="${CTX_CLUSTER2}" \
     -f @samples/helloworld/helloworld.yaml@ \
-    -l app=helloworld -n sample
+    -l service=helloworld -n sample
 {{< /text >}}
 
 ### Deploy `HelloWorld` `V1`
@@ -621,13 +641,13 @@ Deploy the `helloworld-v1` application to `cluster1`:
 {{< text bash >}}
 $ kubectl apply --context="${CTX_CLUSTER1}" \
     -f @samples/helloworld/helloworld.yaml@ \
-    -l app=helloworld -l version=v1 -n sample
+    -l version=v1 -n sample
 {{< /text >}}
 
 Confirm the `helloworld-v1` pod status:
 
 {{< text bash >}}
-$ kubectl get pod --context="${CTX_CLUSTER1}" -n sample
+$ kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l app=helloworld
 NAME                            READY     STATUS    RESTARTS   AGE
 helloworld-v1-86f77cd7bd-cpxhv  2/2       Running   0          40s
 {{< /text >}}
@@ -641,13 +661,13 @@ Deploy the `helloworld-v2` application to `cluster2`:
 {{< text bash >}}
 $ kubectl apply --context="${CTX_CLUSTER2}" \
     -f @samples/helloworld/helloworld.yaml@ \
-    -l app=helloworld -l version=v2 -n sample
+    -l version=v2 -n sample
 {{< /text >}}
 
 Confirm the status the `helloworld-v2` pod status:
 
 {{< text bash >}}
-$ kubectl get pod --context="${CTX_CLUSTER2}" -n sample
+$ kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l app=helloworld
 NAME                            READY     STATUS    RESTARTS   AGE
 helloworld-v2-758dd55874-6x4t8  2/2       Running   0          40s
 {{< /text >}}
@@ -669,6 +689,7 @@ Confirm the status `Sleep` pod on `cluster1`:
 
 {{< text bash >}}
 $ kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l app=sleep
+NAME                             READY   STATUS    RESTARTS   AGE
 sleep-754684654f-n6bzf           2/2     Running   0          5s
 {{< /text >}}
 
@@ -678,6 +699,7 @@ Confirm the status of the `Sleep` pod on `cluster2`:
 
 {{< text bash >}}
 $ kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l app=sleep
+NAME                             READY   STATUS    RESTARTS   AGE
 sleep-754684654f-dzl9j           2/2     Running   0          5s
 {{< /text >}}
 

content/en/docs/setup/install/multicluster/multi-primary-multi-network-test.sh

@@ -25,38 +25,59 @@ source content/en/docs/setup/install/multicluster/common.sh
 set_multi_network_vars
 
 function install_istio_on_cluster1 {
-  snip_install_istio_7
+    echo "Installing Istio on Primary cluster: ${CTX_CLUSTER1}"
-  echo y | snip_install_istio_8
 
-  # Expose services through the east-west gateway.
+    snip_install_istio_7
-  snip_install_istio_9
+    echo y | snip_install_istio_8
-  snip_install_istio_10
+
+    echo "Creating the east-west gateway"
+    snip_install_istio_9
+
+    echo "Waiting for the east-west gateway to have an external IP"
+    _wait_for_gateway_ip istio-system istio-eastwestgateway "${CTX_CLUSTER1}"
+
+    echo "Exposing services via the east-west gateway"
+    snip_install_istio_10
 }
 
 function install_istio_on_cluster2 {
-  snip_install_istio_11
+    echo "Installing Istio on Primary cluster: ${CTX_CLUSTER2}"
-  echo y | snip_install_istio_12
 
-  # Expose services through the east-west gateway.
+    snip_install_istio_11
-  snip_install_istio_13
+    echo y | snip_install_istio_12
-  snip_install_istio_14
+
+    echo "Creating the east-west gateway"
+    snip_install_istio_13
+
+    echo "Exposing services via the east-west gateway"
+    snip_install_istio_14
 }
 
-# Install Istio on the 2 clusters. Executing in
+function install_istio {
-# parallel to reduce test time.
+  # Install Istio on the 2 clusters. Executing in
-install_istio_on_cluster1 &
+  # parallel to reduce test time.
-install_istio_on_cluster2 &
+  install_istio_on_cluster1 &
-wait
+  install_istio_on_cluster2 &
+  wait
+}
 
-# Configure endpoint discovery.
+function configure_endpoint_discovery {
-snip_install_istio_15
+  # Configure endpoint discovery.
-snip_install_istio_16
+  snip_install_istio_15
+  snip_install_istio_16
+}
 
-# Verify that traffic is properly load balanced.
+time configure_trust
-verify_load_balancing
+time install_istio
+time configure_endpoint_discovery
+time verify_load_balancing
 
 # @cleanup
 source content/en/docs/setup/install/multicluster/common.sh
 set +e # ignore cleanup errors
 set_multi_network_vars
-cleanup
+time cleanup
+
+# Everything should be removed once cleanup completes. Use a small
+# number of retries for comparing cluster snapshots before/after the test.
+export VERIFY_RETRIES=1

content/en/docs/setup/install/multicluster/multi-primary-test.sh

@@ -25,30 +25,42 @@ source content/en/docs/setup/install/multicluster/common.sh
 set_single_network_vars
 
 function install_istio_on_cluster1 {
-  snip_install_istio_1
+    echo "Installing Istio on Primary cluster: ${CTX_CLUSTER1}"
-  echo y | snip_install_istio_2
+    snip_install_istio_1
+    echo y | snip_install_istio_2
 }
 
 function install_istio_on_cluster2 {
-  snip_install_istio_3
+    echo "Installing Istio on Primary cluster: ${CTX_CLUSTER2}"
-  echo y | snip_install_istio_4
+    snip_install_istio_3
+    echo y | snip_install_istio_4
 }
 
-# Install Istio on the 2 clusters. Executing in
+function install_istio {
-# parallel to reduce test time.
+  # Install Istio on the 2 clusters. Executing in
-install_istio_on_cluster1 &
+  # parallel to reduce test time.
-install_istio_on_cluster2 &
+  install_istio_on_cluster1 &
-wait
+  install_istio_on_cluster2 &
+  wait
+}
 
-# Configure endpoint discovery.
+function configure_endpoint_discovery {
-snip_install_istio_5
+  # Configure endpoint discovery.
-snip_install_istio_6
+  snip_install_istio_5
+  snip_install_istio_6
+}
 
-# Verify that traffic is properly load balanced.
+time configure_trust
-verify_load_balancing
+time install_istio
+time configure_endpoint_discovery
+time verify_load_balancing
 
 # @cleanup
 source content/en/docs/setup/install/multicluster/common.sh
 set +e # ignore cleanup errors
 set_single_network_vars
-cleanup
+time cleanup
+
+# Everything should be removed once cleanup completes. Use a small
+# number of retries for comparing cluster snapshots before/after the test.
+export VERIFY_RETRIES=1

content/en/docs/setup/install/multicluster/primary-remote-multi-network-test.sh

@@ -25,39 +25,55 @@ source content/en/docs/setup/install/multicluster/common.sh
 set_multi_network_vars
 
 function install_istio_on_cluster1 {
-  snip_install_istio_25
+    echo "Installing Istio on Primary cluster: ${CTX_CLUSTER1}"
-  echo y | snip_install_istio_26
+    snip_install_istio_25
+    echo y | snip_install_istio_26
 
-  # Expose istiod and services via east-west gateway.
+    echo "Creating the east-west gateway"
-  snip_install_istio_27
+    snip_install_istio_27
-  snip_install_istio_28
+
-  snip_install_istio_29
+    echo "Waiting for the east-west gateway to have an external IP"
+    _wait_for_gateway_ip istio-system istio-eastwestgateway "${CTX_CLUSTER1}"
+
+    echo "Exposing istiod via the east-west gateway"
+    snip_install_istio_28
+
+    echo "Exposing services via the east-west gateway"
+    snip_install_istio_29
 }
 
 function install_istio_on_cluster2 {
-  snip_install_istio_30
+    echo "Installing Istio on Remote cluster: ${CTX_CLUSTER2}"
-  snip_install_istio_31
+    snip_install_istio_31
-  echo y | snip_install_istio_32
+    snip_install_istio_32
+    echo y | snip_install_istio_33
 
-  # Expose services via east-west gateway
+    echo "Creating the east-west gateway"
-  snip_install_istio_33
+    snip_install_istio_34
-  snip_install_istio_34
+
+    echo "Waiting for the east-west gateway to have an external IP"
+    _wait_for_gateway_ip istio-system istio-eastwestgateway "${CTX_CLUSTER2}"
+
+    echo "Exposing services via the east-west gateway"
+    snip_install_istio_35
 }
 
-# Install Istio on the 2 clusters. Executing in
+function configure_api_server_access {
-# parallel to reduce test time.
+  snip_install_istio_30
-install_istio_on_cluster1 &
+}
-install_istio_on_cluster2 &
-wait
 
-# Configure endpoint discovery.
+time configure_trust
-snip_install_istio_35
+time install_istio_on_cluster1
-
+time configure_api_server_access
-# Verify that traffic is properly load balanced.
+time install_istio_on_cluster2
-verify_load_balancing
+time verify_load_balancing
 
 # @cleanup
 source content/en/docs/setup/install/multicluster/common.sh
 set +e # ignore cleanup errors
 set_multi_network_vars
-cleanup
+time cleanup
+
+# Everything should be removed once cleanup completes. Use a small
+# number of retries for comparing cluster snapshots before/after the test.
+export VERIFY_RETRIES=1

content/en/docs/setup/install/multicluster/primary-remote-test.sh

@@ -25,34 +25,43 @@ source content/en/docs/setup/install/multicluster/common.sh
 set_single_network_vars
 
 function install_istio_on_cluster1 {
-  snip_install_istio_17
+    echo "Installing Istio on Primary cluster: ${CTX_CLUSTER1}"
-  echo y | snip_install_istio_18
+    snip_install_istio_17
+    echo y | snip_install_istio_18
 
-  # Expose istiod via east-west gateway.
+    echo "Creating the east-west gateway"
-  snip_install_istio_19
+    snip_install_istio_19
-  snip_install_istio_20
+
+    echo "Waiting for the east-west gateway to have an external IP"
+    _wait_for_gateway_ip istio-system istio-eastwestgateway "${CTX_CLUSTER1}"
+
+    echo "Exposing istiod via the east-west gateway"
+    snip_install_istio_20
 }
 
 function install_istio_on_cluster2 {
-  snip_install_istio_21
+    echo "Installing Istio on Remote cluster: ${CTX_CLUSTER2}"
-  snip_install_istio_22
+    snip_install_istio_22
-  echo y | snip_install_istio_23
+    snip_install_istio_23
+    echo y | snip_install_istio_24
 }
 
-# Install Istio on the 2 clusters. Executing in
+function configure_api_server_access {
-# parallel to reduce test time.
+  snip_install_istio_21
-install_istio_on_cluster1 &
+}
-install_istio_on_cluster2 &
-wait
 
-# Configure endpoint discovery.
+time configure_trust
-snip_install_istio_24
+time install_istio_on_cluster1
-
+time configure_api_server_access
-# Verify that traffic is properly load balanced.
+time install_istio_on_cluster2
-verify_load_balancing
+time verify_load_balancing
 
 # @cleanup
 source content/en/docs/setup/install/multicluster/common.sh
 set +e # ignore cleanup errors
 set_single_network_vars
-cleanup
+time cleanup
+
+# Everything should be removed once cleanup completes. Use a small
+# number of retries for comparing cluster snapshots before/after the test.
+export VERIFY_RETRIES=1

content/en/docs/setup/install/multicluster/snips.sh

@@ -26,7 +26,7 @@ export CTX_CLUSTER2=cluster2
 }
 
 snip_install_istio_1() {
-cat <<EOF > ./cluster1.yaml
+cat <<EOF > cluster1.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -44,7 +44,7 @@ istioctl install --context="${CTX_CLUSTER1}" -f cluster1.yaml
 }
 
 snip_install_istio_3() {
-cat <<EOF > ./cluster2.yaml
+cat <<EOF > cluster2.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -76,7 +76,7 @@ istioctl x create-remote-secret \
 }
 
 snip_install_istio_7() {
-cat <<EOF > ./cluster1.yaml
+cat <<EOF > cluster1.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -106,7 +106,7 @@ kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \
 }
 
 snip_install_istio_11() {
-cat <<EOF > ./cluster2.yaml
+cat <<EOF > cluster2.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -150,7 +150,7 @@ istioctl x create-remote-secret \
 }
 
 snip_install_istio_17() {
-cat <<EOF > ./cluster1.yaml
+cat <<EOF > cluster1.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -180,14 +180,21 @@ kubectl apply --context="${CTX_CLUSTER1}" -f \
 }
 
 snip_install_istio_21() {
+istioctl x create-remote-secret \
+    --context="${CTX_CLUSTER2}" \
+    --name=cluster2 | \
+    kubectl apply -f - --context="${CTX_CLUSTER1}"
+}
+
+snip_install_istio_22() {
 export DISCOVERY_ADDRESS=$(kubectl \
     --context="${CTX_CLUSTER1}" \
     -n istio-system get svc istio-eastwestgateway \
     -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
 }
 
-snip_install_istio_22() {
+snip_install_istio_23() {
-cat <<EOF > ./cluster2.yaml
+cat <<EOF > cluster2.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -201,19 +208,12 @@ spec:
 EOF
 }
 
-snip_install_istio_23() {
+snip_install_istio_24() {
 istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml
 }
 
-snip_install_istio_24() {
-istioctl x create-remote-secret \
-    --context="${CTX_CLUSTER2}" \
-    --name=cluster2 | \
-    kubectl apply -f - --context="${CTX_CLUSTER1}"
-}
-
 snip_install_istio_25() {
-cat <<EOF > ./cluster1.yaml
+cat <<EOF > cluster1.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -248,14 +248,21 @@ kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \
 }
 
 snip_install_istio_30() {
+istioctl x create-remote-secret \
+    --context="${CTX_CLUSTER2}" \
+    --name=cluster2 | \
+    kubectl apply -f - --context="${CTX_CLUSTER1}"
+}
+
+snip_install_istio_31() {
 export DISCOVERY_ADDRESS=$(kubectl \
     --context="${CTX_CLUSTER1}" \
     -n istio-system get svc istio-eastwestgateway \
     -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
 }
 
-snip_install_istio_31() {
+snip_install_istio_32() {
-cat <<EOF > ./cluster2.yaml
+cat <<EOF > cluster2.yaml
 apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
@@ -269,29 +276,22 @@ spec:
 EOF
 }
 
-snip_install_istio_32() {
+snip_install_istio_33() {
 istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml
 }
 
-snip_install_istio_33() {
+snip_install_istio_34() {
 MESH=mesh1 CLUSTER=cluster2 NETWORK=network2 \
     samples/multicluster/gen-eastwest-gateway.sh | \
     istioctl manifest generate -f - | \
     kubectl apply --context="${CTX_CLUSTER2}" -f -
 }
 
-snip_install_istio_34() {
+snip_install_istio_35() {
 kubectl --context="${CTX_CLUSTER2}" apply -n istio-system -f \
     samples/multicluster/expose-services.yaml
 }
 
-snip_install_istio_35() {
-istioctl x create-remote-secret \
-    --context="${CTX_CLUSTER2}" \
-    --name=cluster2 | \
-    kubectl apply -f - --context="${CTX_CLUSTER1}"
-}
-
 snip_deploy_the_helloworld_service_1() {
 kubectl create --context="${CTX_CLUSTER1}" namespace sample
 kubectl create --context="${CTX_CLUSTER2}" namespace sample
@@ -307,20 +307,20 @@ kubectl label --context="${CTX_CLUSTER2}" namespace sample \
 snip_deploy_the_helloworld_service_3() {
 kubectl apply --context="${CTX_CLUSTER1}" \
     -f samples/helloworld/helloworld.yaml \
-    -l app=helloworld -n sample
+    -l service=helloworld -n sample
 kubectl apply --context="${CTX_CLUSTER2}" \
     -f samples/helloworld/helloworld.yaml \
-    -l app=helloworld -n sample
+    -l service=helloworld -n sample
 }
 
 snip_deploy_helloworld_v1_1() {
 kubectl apply --context="${CTX_CLUSTER1}" \
     -f samples/helloworld/helloworld.yaml \
-    -l app=helloworld -l version=v1 -n sample
+    -l version=v1 -n sample
 }
 
 snip_deploy_helloworld_v1_2() {
-kubectl get pod --context="${CTX_CLUSTER1}" -n sample
+kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l app=helloworld
 }
 
 ! read -r -d '' snip_deploy_helloworld_v1_2_out <<\ENDSNIP
@@ -331,11 +331,11 @@ ENDSNIP
 snip_deploy_helloworld_v2_1() {
 kubectl apply --context="${CTX_CLUSTER2}" \
     -f samples/helloworld/helloworld.yaml \
-    -l app=helloworld -l version=v2 -n sample
+    -l version=v2 -n sample
 }
 
 snip_deploy_helloworld_v2_2() {
-kubectl get pod --context="${CTX_CLUSTER2}" -n sample
+kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l app=helloworld
 }
 
 ! read -r -d '' snip_deploy_helloworld_v2_2_out <<\ENDSNIP
@@ -355,6 +355,7 @@ kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l app=sleep
 }
 
 ! read -r -d '' snip_deploy_sleep_2_out <<\ENDSNIP
+NAME                             READY   STATUS    RESTARTS   AGE
 sleep-754684654f-n6bzf           2/2     Running   0          5s
 ENDSNIP
 
@@ -363,6 +364,7 @@ kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l app=sleep
 }
 
 ! read -r -d '' snip_deploy_sleep_3_out <<\ENDSNIP
+NAME                             READY   STATUS    RESTARTS   AGE
 sleep-754684654f-dzl9j           2/2     Running   0          5s
 ENDSNIP
 

go.mod

@@ -27,6 +27,6 @@ replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.5
 replace github.com/evanphx/json-patch => github.com/evanphx/json-patch v0.0.0-20190815234213-e83c0a1c26c8
 
 require (
-	istio.io/istio v0.0.0-20201014161623-394493e824eb
+	istio.io/istio v0.0.0-20201020214016-6ba5c6b9a1e7
 	istio.io/pkg v0.0.0-20200922180714-670b76a68558
 )

go.sum

@@ -1233,15 +1233,15 @@ honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 istio.io/api v0.0.0-20190515205759-982e5c3888c6/go.mod h1:hhLFQmpHia8zgaM37vb2ml9iS5NfNfqZGRt1pS9aVEo=
 istio.io/api v0.0.0-20200812202721-24be265d41c3/go.mod h1:88HN3o1fSD1jo+Z1WTLlJfMm9biopur6Ct9BFKjiB64=
-istio.io/api v0.0.0-20201007205536-38d3c76a5557 h1:EDgHyV3kq988v2vb4p4j/D0saXf1JGh43uWhEkD4x+M=
+istio.io/api v0.0.0-20201020011337-0d3a960deddb h1:/oFtAb7roeWXlZh9CDWM4pQsPeDfunzxVg4aldSLw84=
-istio.io/api v0.0.0-20201007205536-38d3c76a5557/go.mod h1:88HN3o1fSD1jo+Z1WTLlJfMm9biopur6Ct9BFKjiB64=
+istio.io/api v0.0.0-20201020011337-0d3a960deddb/go.mod h1:88HN3o1fSD1jo+Z1WTLlJfMm9biopur6Ct9BFKjiB64=
 istio.io/client-go v0.0.0-20200908160912-f99162621a1a h1:clPn0fz+rXq5Ytj6Ppb1ygUKeU0RImT4ZbT1oMd1G04=
 istio.io/client-go v0.0.0-20200908160912-f99162621a1a/go.mod h1:SO65MWt7I45dvUwuDowoiB0SVcGpfWZfUTlopvYpbZc=
 istio.io/gogo-genproto v0.0.0-20190930162913-45029607206a/go.mod h1:OzpAts7jljZceG4Vqi5/zXy/pOg1b209T3jb7Nv5wIs=
 istio.io/gogo-genproto v0.0.0-20200908160912-66171252e3db h1:btPv5pCusFxbWrmiLNLEThH8IzWunUeZ1r5M1O22vBc=
 istio.io/gogo-genproto v0.0.0-20200908160912-66171252e3db/go.mod h1:OzpAts7jljZceG4Vqi5/zXy/pOg1b209T3jb7Nv5wIs=
-istio.io/istio v0.0.0-20201014161623-394493e824eb h1:TtFRcM8eYDPvZOArA1Ykne9JMzXSPMPBOqVnoT/oUUM=
+istio.io/istio v0.0.0-20201020214016-6ba5c6b9a1e7 h1:6CJDE2X7mH6pcNpNNniD8r9ZSnDZn7xo6Ay3VewfkJ0=
-istio.io/istio v0.0.0-20201014161623-394493e824eb/go.mod h1:bOY51cI9r95WV7gSP4r/bWzaxcCjaXuNGBvLEJQ0wJc=
+istio.io/istio v0.0.0-20201020214016-6ba5c6b9a1e7/go.mod h1:nPwhi1qIogLf4pxXfdh86h+A4TM4SU5JfxSsvroaXIQ=
 istio.io/pkg v0.0.0-20200922180714-670b76a68558 h1:ATDshla3gX4b0X8oMs0rrFry9qJshrzTnFkBV42CcFk=
 istio.io/pkg v0.0.0-20200922180714-670b76a68558/go.mod h1:p6wktGBjkjL3spRSsyfOh0XkuKb8IuBX61rERHfmSbU=
 k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8=
@@ -1250,10 +1250,10 @@ k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78=
 k8s.io/api v0.18.3 h1:2AJaUQdgUZLoDZHrun21PW2Nx9+ll6cUzvn3IKhSIn0=
 k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA=
 k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI=
-k8s.io/api v0.19.0 h1:XyrFIJqTYZJ2DU7FBE/bSPz7b1HvbVBuBf07oeo6eTc=
-k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw=
 k8s.io/api v0.19.1 h1:oZf4bYsBdjC49PdTwNfLmrfUFCwKUi94HY/+emXI8Qw=
 k8s.io/api v0.19.1/go.mod h1:+u/k4/K/7vp4vsfdT7dyl8Oxk1F26Md4g5F26Tu85PU=
+k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms=
+k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI=
 k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo=
 k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY=
 k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M=
@@ -1265,10 +1265,10 @@ k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftc
 k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk=
 k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
 k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
-k8s.io/apimachinery v0.19.0 h1:gjKnAda/HZp5k4xQYjL0K/Yb66IvNqjthCb03QlKpaQ=
-k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
 k8s.io/apimachinery v0.19.1 h1:cwsxZazM/LA9aUsBaL4bRS5ygoM6bYp8dFk22DSYQa4=
 k8s.io/apimachinery v0.19.1/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
+k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc=
+k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
 k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw=
 k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw=
 k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg=
@@ -1280,15 +1280,16 @@ k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8=
 k8s.io/client-go v0.18.1/go.mod h1:iCikYRiXOj/yRRFE/aWqrpPtDt4P2JVWhtHkmESTcfY=
 k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU=
 k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q=
-k8s.io/client-go v0.19.0 h1:1+0E0zfWFIWeyRhQYWzimJOyAk2UT7TiARaLNwJCf7k=
-k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU=
 k8s.io/client-go v0.19.1 h1:xfFwj+YFKa8rcihlFYZABjxcy7Sm/wJQ+GxW3JyVtKI=
 k8s.io/client-go v0.19.1/go.mod h1:AZOIVSI9UUtQPeJD3zJFp15CEhSjRgAuQP5PWRJrCIQ=
+k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc=
+k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA=
 k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc=
 k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc=
 k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
 k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
 k8s.io/code-generator v0.19.1/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
+k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
 k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c=
 k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM=
 k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14=
@@ -1333,8 +1334,8 @@ sigs.k8s.io/controller-runtime v0.6.2/go.mod h1:vhcq/rlnENJ09SIRp3EveTaZ0yqH526h
 sigs.k8s.io/controller-tools v0.4.0/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU=
 sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
-sigs.k8s.io/service-apis v0.0.0-20200916220245-b060b8df63c9 h1:9z7RSE6mFtwzhHvuZWqki6KSLp9g94jt17ysvft/nEU=
+sigs.k8s.io/service-apis v0.0.0-20201013190827-799dbbe3a3ba h1:tmIsMOyPg1eOSRDFMinrHktWafHyosZcxK/hnFS5HFo=
-sigs.k8s.io/service-apis v0.0.0-20200916220245-b060b8df63c9/go.mod h1:yfIjKFwmoJxpP4m4qOmdBLgJvlSw61GDYuKTuRo7O1c=
+sigs.k8s.io/service-apis v0.0.0-20201013190827-799dbbe3a3ba/go.mod h1:QkiV/PnK7YbN5zqYqXnh5wByTTT1LYJ5scwdIs62qWs=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=

tests/util/helpers.sh

@@ -70,16 +70,50 @@ _set_ingress_environment_variables() {
 #   kubectl wait --for=condition=Ready pod --all --timeout=60s
 
 # Wait for rollout of named deployment
-# usage: _wait_for_deployment <namespace> <deployment name>
+# usage: _wait_for_deployment <namespace> <deployment name> <optional: context>
 _wait_for_deployment() {
     local namespace="$1"
     local name="$2"
-    if ! kubectl -n "$namespace" rollout status deployment "$name" --timeout 5m; then
+    local context="${3:-}"
+    if ! kubectl --context="$context" -n "$namespace" rollout status deployment "$name" --timeout 5m; then
         echo "Failed rollout of deployment $name in namespace $namespace"
         exit 1
     fi
 }
 
+# Wait for the given gateway to be allocated an external IP.
+# usage: _wait_for_gateway_ip <namespace> <service name> <optional: context>
+_wait_for_gateway_ip() {
+    local namespace="$1"
+    local service="$2"
+    local context="${3:-}"
+
+    local max_time=${MAX_SECONDS:-300} # Default to 5 min.
+    local delay=5
+
+    local start_time=$(date +%s)
+    local current_time=$start_time
+    local end_time=$((start_time + max_time))
+
+    while true; do
+        local ip=$(kubectl --context="${context}" get svc "${service}" -n "${namespace}" -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+
+        # Verify that the IP is set.
+        if [[ -n "${ip}" ]]; then
+            echo "IP Assigned for $service.$namespace: ${ip}"
+            return
+        fi
+
+        current_time=$(date +%s)
+        if (( current_time > end_time )); then
+            echo "Failed waiting for $service.$namespace: ${ip}"
+            exit 1
+        fi
+
+        sleep "${delay}"
+    done
+}
+
 # Wait for Istio config to propagate
 # usage: _wait_for_istio <kind> <namespace> <name>
 _wait_for_istio() {

tests/util/verify.sh

@@ -313,6 +313,8 @@ __create_cluster_snapshots() {
 }
 
 __cluster_cleanup_check() {
+    VERIFY_RETRIES=${VERIFY_RETRIES:-9}
+
     # Get the list of KUBECONFIG files as an array.
     IFS=':' read -r -a KFILES <<< "${KUBECONFIG}"
     for KFILE in "${KFILES[@]}"; do
@@ -330,7 +332,6 @@ __cluster_cleanup_check() {
         rm "${SNAPSHOT_FILE}"
 
         # Verify that we've restored the original cluster state.
-        VERIFY_RETRIES=9
         (KUBECONFIG="${KFILE}"; _verify_like __cluster_state "${SNAPSHOT}")
         echo "Finished cleanup check against snapshot ${SNAPSHOT_FILE}"
     done