diff --git a/content/en/docs/setup/install/multicluster/common.sh b/content/en/docs/setup/install/multicluster/common.sh index b6341e6c61..95c86ce3c7 100644 --- a/content/en/docs/setup/install/multicluster/common.sh +++ b/content/en/docs/setup/install/multicluster/common.sh @@ -36,11 +36,44 @@ function set_multi_network_vars export CTX_CLUSTER2="${KUBE_CONTEXTS[2]}" } -# cleanup removes all resources created by the tests. -function cleanup() +# configure_trust creates a hierarchy of +function configure_trust { - # Remove generated yaml files. - rm -f "cluster1.yaml" "cluster2.yaml" + # Keeps the certs under a separate directory. + mkdir -p certs + pushd certs || exit + + # Create the root cert. + make -f ../tools/certs/Makefile.selfsigned.mk root-ca + + # Create and deploy intermediate certs for cluster1 and cluster2. + make -f ../tools/certs/Makefile.selfsigned.mk cluster1-cacerts + make -f ../tools/certs/Makefile.selfsigned.mk cluster2-cacerts + + # Create the istio-system namespace in each cluster so that we can create the secrets. + kubectl --context="$CTX_CLUSTER1" create namespace istio-system + kubectl --context="$CTX_CLUSTER2" create namespace istio-system + + # Deploy secret to each cluster + kubectl --context="$CTX_CLUSTER1" create secret generic cacerts -n istio-system \ + --from-file=cluster1/ca-cert.pem \ + --from-file=cluster1/ca-key.pem \ + --from-file=cluster1/root-cert.pem \ + --from-file=cluster1/cert-chain.pem + kubectl --context="$CTX_CLUSTER2" create secret generic cacerts -n istio-system \ + --from-file=cluster2/ca-cert.pem \ + --from-file=cluster2/ca-key.pem \ + --from-file=cluster2/root-cert.pem \ + --from-file=cluster2/cert-chain.pem + + popd || exit # Return to the previous directory. +} + +# cleanup removes all resources created by the tests. +function cleanup +{ + # Remove temp files. + rm -f cluster1.yaml cluster2.yaml certs # Delete the namespaces on both clusters concurrently delete_namespaces_cluster1 & @@ -50,47 +83,62 @@ function cleanup() # _delete_namespaces_cluster1 removes the istio-system and sample namespaces on both # CLUSTER1. -function delete_namespaces_cluster1() +function delete_namespaces_cluster1 { kubectl delete ns istio-system sample --context="${CTX_CLUSTER1}" --ignore-not-found } # _delete_namespaces_cluster2 removes the istio-system and sample namespaces on both # CLUSTER2. -function delete_namespaces_cluster2() +function delete_namespaces_cluster2 { kubectl delete ns istio-system sample --context="${CTX_CLUSTER2}" --ignore-not-found } # verify_load_balancing verifies that traffic is load balanced properly # between CLUSTER1 and CLUSTER2. -function verify_load_balancing() +function verify_load_balancing { - # Deploy HelloWorld and Sleep. + # Deploy the HelloWorld service. snip_deploy_the_helloworld_service_1 snip_deploy_the_helloworld_service_2 snip_deploy_the_helloworld_service_3 + + # Deploy HelloWorld v1 and v2 snip_deploy_helloworld_v1_1 snip_deploy_helloworld_v2_1 + + # Deploy Sleep snip_deploy_sleep_1 - snip_deploy_sleep_3 - # Wait for the deployments in CLUSTER1 - (KUBECONFIG="${KUBECONFIG_CLUSTER1}"; _wait_for_deployment sample helloworld-v1) - (KUBECONFIG="${KUBECONFIG_CLUSTER1}"; _wait_for_deployment sample sleep) + # Wait for all the deployments. + _wait_for_deployment sample helloworld-v1 "${CTX_CLUSTER1}" + _wait_for_deployment sample sleep "${CTX_CLUSTER1}" + _wait_for_deployment sample helloworld-v2 "${CTX_CLUSTER2}" + _wait_for_deployment sample sleep "${CTX_CLUSTER2}" - # Wait for the deployments in CLUSTER2 - (KUBECONFIG="${KUBECONFIG_CLUSTER2}"; _wait_for_deployment sample helloworld-v2) - (KUBECONFIG="${KUBECONFIG_CLUSTER2}"; _wait_for_deployment sample sleep) + # Verify everything is deployed as expected. + VERIFY_RETRIES=0 # Don't retry. + echo "Verifying helloworld v1 deployment" + _verify_like snip_deploy_helloworld_v1_2 "$snip_deploy_helloworld_v1_2_out" + echo "Verifying helloworld v2 deployment" + _verify_like snip_deploy_helloworld_v2_2 "$snip_deploy_helloworld_v2_2_out" + echo "Verifying sleep deployment in ${CTX_CLUSTER1}" + _verify_like snip_deploy_sleep_2 "$snip_deploy_sleep_2_out" + echo "Verifying sleep deployment in ${CTX_CLUSTER2}" + _verify_like snip_deploy_sleep_3 "$snip_deploy_sleep_3_out" + unset VERIFY_RETRIES # Restore default local EXPECTED_RESPONSE_FROM_CLUSTER1="Hello version: v1, instance:" local EXPECTED_RESPONSE_FROM_CLUSTER2="Hello version: v2, instance:" # Verify we hit both clusters from CLUSTER1 + echo "Verifying load balancing from ${CTX_CLUSTER1}" _verify_contains snip_verifying_crosscluster_traffic_1 "$EXPECTED_RESPONSE_FROM_CLUSTER1" _verify_contains snip_verifying_crosscluster_traffic_1 "$EXPECTED_RESPONSE_FROM_CLUSTER2" # Verify we hit both clusters from CLUSTER2 + echo "Verifying load balancing from ${CTX_CLUSTER2}" _verify_contains snip_verifying_crosscluster_traffic_3 "$EXPECTED_RESPONSE_FROM_CLUSTER1" _verify_contains snip_verifying_crosscluster_traffic_3 "$EXPECTED_RESPONSE_FROM_CLUSTER2" } diff --git a/content/en/docs/setup/install/multicluster/index.md b/content/en/docs/setup/install/multicluster/index.md index 6c1c95cf47..52e64696c9 100644 --- a/content/en/docs/setup/install/multicluster/index.md +++ b/content/en/docs/setup/install/multicluster/index.md @@ -112,7 +112,7 @@ Service workloads communicate directly (pod-to-pod) across cluster boundaries. Create the Istio configuration for `cluster1`: {{< text bash >}} -$ cat < ./cluster1.yaml +$ cat < cluster1.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -136,7 +136,7 @@ $ istioctl install --context="${CTX_CLUSTER1}" -f cluster1.yaml Create the Istio configuration for `cluster2`: {{< text bash >}} -$ cat < ./cluster2.yaml +$ cat < cluster2.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -200,7 +200,7 @@ traffic. The gateway in each cluster must be reachable from the other cluster. Create the Istio configuration for `cluster1`: {{< text bash >}} -$ cat < ./cluster1.yaml +$ cat < cluster1.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -249,7 +249,7 @@ $ kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \ Create the Istio configuration for `cluster2`: {{< text bash >}} -$ cat < ./cluster2.yaml +$ cat < cluster2.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -333,7 +333,7 @@ traffic. Create the Istio configuration for `cluster1`: {{< text bash >}} -$ cat < ./cluster1.yaml +$ cat < cluster1.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -374,6 +374,28 @@ $ kubectl apply --context="${CTX_CLUSTER1}" -f \ @samples/multicluster/expose-istiod.yaml@ {{< /text >}} +

Enable API Server Access to cluster2

+ +Before we can configure the remote cluster, we first have to give the control +plane in `cluster1` access to the API Server in `cluster2`. This will do the +following: + +- Enables the control plane to authenticate connection requests from + workloads running in `cluster2`. Without API Server access, the control + plane will reject the requests. + +- Enables discovery of service endpoints running in `cluster2`. + +To provide API Server access to `cluster2`, we generate a remote secret and +apply it to `cluster1`: + +{{< text bash >}} +$ istioctl x create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + --name=cluster2 | \ + kubectl apply -f - --context="${CTX_CLUSTER1}" +{{< /text >}} +

Configure cluster2 as a remote

Save the address of `cluster1`’s ingress gateway. @@ -388,7 +410,7 @@ $ export DISCOVERY_ADDRESS=$(kubectl \ Now create a remote configuration for `cluster2`. {{< text bash >}} -$ cat < ./cluster2.yaml +$ cat < cluster2.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -408,18 +430,6 @@ Apply the configuration to `cluster2`: $ istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml {{< /text >}} -

Enable Endpoint Discovery for cluster2

- -Create a remote secret that will allow the control plane in `cluster1` to access the -API Server in `cluster2` for endpoints. - -{{< text bash >}} -$ istioctl x create-remote-secret \ - --context="${CTX_CLUSTER2}" \ - --name=cluster2 | \ - kubectl apply -f - --context="${CTX_CLUSTER1}" -{{< /text >}} - {{< /tab >}} {{< tab name="Primary-Remote, Multi-Network" category-value="primary-remote-multi-network" >}} @@ -452,7 +462,7 @@ same east-west gateway. Create the Istio configuration for `cluster1`: {{< text bash >}} -$ cat < ./cluster1.yaml +$ cat < cluster1.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -503,6 +513,28 @@ $ kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \ @samples/multicluster/expose-services.yaml@ {{< /text >}} +

Enable API Server Access to cluster2

+ +Before we can configure the remote cluster, we first have to give the control +plane in `cluster1` access to the API Server in `cluster2`. This will do the +following: + +- Enables the control plane to authenticate connection requests from + workloads running in `cluster2`. Without API Server access, the control + plane will reject the requests. + +- Enables discovery of service endpoints running in `cluster2`. + +To provide API Server access to `cluster2`, we generate a remote secret and +apply it to `cluster1`: + +{{< text bash >}} +$ istioctl x create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + --name=cluster2 | \ + kubectl apply -f - --context="${CTX_CLUSTER1}" +{{< /text >}} +

Configure cluster2 as a remote with services exposed

Save the address of `cluster1`’s ingress gateway. @@ -517,7 +549,7 @@ $ export DISCOVERY_ADDRESS=$(kubectl \ Now create a remote configuration on `cluster2`. {{< text bash >}} -$ cat < ./cluster2.yaml +$ cat < cluster2.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -552,18 +584,6 @@ $ kubectl --context="${CTX_CLUSTER2}" apply -n istio-system -f \ @samples/multicluster/expose-services.yaml@ {{< /text >}} -

Enable Endpoint Discovery for cluster2 on network2

- -Create a remote secret that will allow the control plane in `cluster1` to -access the API Server in `cluster2` for endpoints. - -{{< text bash >}} -$ istioctl x create-remote-secret \ - --context="${CTX_CLUSTER2}" \ - --name=cluster2 | \ - kubectl apply -f - --context="${CTX_CLUSTER1}" -{{< /text >}} - {{< /tab >}} {{< /tabset >}} @@ -608,10 +628,10 @@ Create the `HelloWorld` service in both clusters: {{< text bash >}} $ kubectl apply --context="${CTX_CLUSTER1}" \ -f @samples/helloworld/helloworld.yaml@ \ - -l app=helloworld -n sample + -l service=helloworld -n sample $ kubectl apply --context="${CTX_CLUSTER2}" \ -f @samples/helloworld/helloworld.yaml@ \ - -l app=helloworld -n sample + -l service=helloworld -n sample {{< /text >}} ### Deploy `HelloWorld` `V1` @@ -621,13 +641,13 @@ Deploy the `helloworld-v1` application to `cluster1`: {{< text bash >}} $ kubectl apply --context="${CTX_CLUSTER1}" \ -f @samples/helloworld/helloworld.yaml@ \ - -l app=helloworld -l version=v1 -n sample + -l version=v1 -n sample {{< /text >}} Confirm the `helloworld-v1` pod status: {{< text bash >}} -$ kubectl get pod --context="${CTX_CLUSTER1}" -n sample +$ kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l app=helloworld NAME READY STATUS RESTARTS AGE helloworld-v1-86f77cd7bd-cpxhv 2/2 Running 0 40s {{< /text >}} @@ -641,13 +661,13 @@ Deploy the `helloworld-v2` application to `cluster2`: {{< text bash >}} $ kubectl apply --context="${CTX_CLUSTER2}" \ -f @samples/helloworld/helloworld.yaml@ \ - -l app=helloworld -l version=v2 -n sample + -l version=v2 -n sample {{< /text >}} Confirm the status the `helloworld-v2` pod status: {{< text bash >}} -$ kubectl get pod --context="${CTX_CLUSTER2}" -n sample +$ kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l app=helloworld NAME READY STATUS RESTARTS AGE helloworld-v2-758dd55874-6x4t8 2/2 Running 0 40s {{< /text >}} @@ -669,6 +689,7 @@ Confirm the status `Sleep` pod on `cluster1`: {{< text bash >}} $ kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l app=sleep +NAME READY STATUS RESTARTS AGE sleep-754684654f-n6bzf 2/2 Running 0 5s {{< /text >}} @@ -678,6 +699,7 @@ Confirm the status of the `Sleep` pod on `cluster2`: {{< text bash >}} $ kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l app=sleep +NAME READY STATUS RESTARTS AGE sleep-754684654f-dzl9j 2/2 Running 0 5s {{< /text >}} diff --git a/content/en/docs/setup/install/multicluster/multi-primary-multi-network-test.sh b/content/en/docs/setup/install/multicluster/multi-primary-multi-network-test.sh index 94d0bd6df1..04225633b6 100644 --- a/content/en/docs/setup/install/multicluster/multi-primary-multi-network-test.sh +++ b/content/en/docs/setup/install/multicluster/multi-primary-multi-network-test.sh @@ -25,38 +25,59 @@ source content/en/docs/setup/install/multicluster/common.sh set_multi_network_vars function install_istio_on_cluster1 { - snip_install_istio_7 - echo y | snip_install_istio_8 + echo "Installing Istio on Primary cluster: ${CTX_CLUSTER1}" - # Expose services through the east-west gateway. - snip_install_istio_9 - snip_install_istio_10 + snip_install_istio_7 + echo y | snip_install_istio_8 + + echo "Creating the east-west gateway" + snip_install_istio_9 + + echo "Waiting for the east-west gateway to have an external IP" + _wait_for_gateway_ip istio-system istio-eastwestgateway "${CTX_CLUSTER1}" + + echo "Exposing services via the east-west gateway" + snip_install_istio_10 } function install_istio_on_cluster2 { - snip_install_istio_11 - echo y | snip_install_istio_12 + echo "Installing Istio on Primary cluster: ${CTX_CLUSTER2}" - # Expose services through the east-west gateway. - snip_install_istio_13 - snip_install_istio_14 + snip_install_istio_11 + echo y | snip_install_istio_12 + + echo "Creating the east-west gateway" + snip_install_istio_13 + + echo "Exposing services via the east-west gateway" + snip_install_istio_14 } -# Install Istio on the 2 clusters. Executing in -# parallel to reduce test time. -install_istio_on_cluster1 & -install_istio_on_cluster2 & -wait +function install_istio { + # Install Istio on the 2 clusters. Executing in + # parallel to reduce test time. + install_istio_on_cluster1 & + install_istio_on_cluster2 & + wait +} -# Configure endpoint discovery. -snip_install_istio_15 -snip_install_istio_16 +function configure_endpoint_discovery { + # Configure endpoint discovery. + snip_install_istio_15 + snip_install_istio_16 +} -# Verify that traffic is properly load balanced. -verify_load_balancing +time configure_trust +time install_istio +time configure_endpoint_discovery +time verify_load_balancing # @cleanup source content/en/docs/setup/install/multicluster/common.sh set +e # ignore cleanup errors set_multi_network_vars -cleanup +time cleanup + +# Everything should be removed once cleanup completes. Use a small +# number of retries for comparing cluster snapshots before/after the test. +export VERIFY_RETRIES=1 diff --git a/content/en/docs/setup/install/multicluster/multi-primary-test.sh b/content/en/docs/setup/install/multicluster/multi-primary-test.sh index ddbab6e6b3..d2de51a4a4 100755 --- a/content/en/docs/setup/install/multicluster/multi-primary-test.sh +++ b/content/en/docs/setup/install/multicluster/multi-primary-test.sh @@ -25,30 +25,42 @@ source content/en/docs/setup/install/multicluster/common.sh set_single_network_vars function install_istio_on_cluster1 { - snip_install_istio_1 - echo y | snip_install_istio_2 + echo "Installing Istio on Primary cluster: ${CTX_CLUSTER1}" + snip_install_istio_1 + echo y | snip_install_istio_2 } function install_istio_on_cluster2 { - snip_install_istio_3 - echo y | snip_install_istio_4 + echo "Installing Istio on Primary cluster: ${CTX_CLUSTER2}" + snip_install_istio_3 + echo y | snip_install_istio_4 } -# Install Istio on the 2 clusters. Executing in -# parallel to reduce test time. -install_istio_on_cluster1 & -install_istio_on_cluster2 & -wait +function install_istio { + # Install Istio on the 2 clusters. Executing in + # parallel to reduce test time. + install_istio_on_cluster1 & + install_istio_on_cluster2 & + wait +} -# Configure endpoint discovery. -snip_install_istio_5 -snip_install_istio_6 +function configure_endpoint_discovery { + # Configure endpoint discovery. + snip_install_istio_5 + snip_install_istio_6 +} -# Verify that traffic is properly load balanced. -verify_load_balancing +time configure_trust +time install_istio +time configure_endpoint_discovery +time verify_load_balancing # @cleanup source content/en/docs/setup/install/multicluster/common.sh set +e # ignore cleanup errors set_single_network_vars -cleanup +time cleanup + +# Everything should be removed once cleanup completes. Use a small +# number of retries for comparing cluster snapshots before/after the test. +export VERIFY_RETRIES=1 diff --git a/content/en/docs/setup/install/multicluster/primary-remote-multi-network-test.sh b/content/en/docs/setup/install/multicluster/primary-remote-multi-network-test.sh index cc6cba6552..bbcbbfb388 100644 --- a/content/en/docs/setup/install/multicluster/primary-remote-multi-network-test.sh +++ b/content/en/docs/setup/install/multicluster/primary-remote-multi-network-test.sh @@ -25,39 +25,55 @@ source content/en/docs/setup/install/multicluster/common.sh set_multi_network_vars function install_istio_on_cluster1 { - snip_install_istio_25 - echo y | snip_install_istio_26 + echo "Installing Istio on Primary cluster: ${CTX_CLUSTER1}" + snip_install_istio_25 + echo y | snip_install_istio_26 - # Expose istiod and services via east-west gateway. - snip_install_istio_27 - snip_install_istio_28 - snip_install_istio_29 + echo "Creating the east-west gateway" + snip_install_istio_27 + + echo "Waiting for the east-west gateway to have an external IP" + _wait_for_gateway_ip istio-system istio-eastwestgateway "${CTX_CLUSTER1}" + + echo "Exposing istiod via the east-west gateway" + snip_install_istio_28 + + echo "Exposing services via the east-west gateway" + snip_install_istio_29 } function install_istio_on_cluster2 { - snip_install_istio_30 - snip_install_istio_31 - echo y | snip_install_istio_32 + echo "Installing Istio on Remote cluster: ${CTX_CLUSTER2}" + snip_install_istio_31 + snip_install_istio_32 + echo y | snip_install_istio_33 - # Expose services via east-west gateway - snip_install_istio_33 - snip_install_istio_34 + echo "Creating the east-west gateway" + snip_install_istio_34 + + echo "Waiting for the east-west gateway to have an external IP" + _wait_for_gateway_ip istio-system istio-eastwestgateway "${CTX_CLUSTER2}" + + echo "Exposing services via the east-west gateway" + snip_install_istio_35 } -# Install Istio on the 2 clusters. Executing in -# parallel to reduce test time. -install_istio_on_cluster1 & -install_istio_on_cluster2 & -wait +function configure_api_server_access { + snip_install_istio_30 +} -# Configure endpoint discovery. -snip_install_istio_35 - -# Verify that traffic is properly load balanced. -verify_load_balancing +time configure_trust +time install_istio_on_cluster1 +time configure_api_server_access +time install_istio_on_cluster2 +time verify_load_balancing # @cleanup source content/en/docs/setup/install/multicluster/common.sh set +e # ignore cleanup errors set_multi_network_vars -cleanup +time cleanup + +# Everything should be removed once cleanup completes. Use a small +# number of retries for comparing cluster snapshots before/after the test. +export VERIFY_RETRIES=1 diff --git a/content/en/docs/setup/install/multicluster/primary-remote-test.sh b/content/en/docs/setup/install/multicluster/primary-remote-test.sh index b296a9f4c0..5320397058 100644 --- a/content/en/docs/setup/install/multicluster/primary-remote-test.sh +++ b/content/en/docs/setup/install/multicluster/primary-remote-test.sh @@ -25,34 +25,43 @@ source content/en/docs/setup/install/multicluster/common.sh set_single_network_vars function install_istio_on_cluster1 { - snip_install_istio_17 - echo y | snip_install_istio_18 + echo "Installing Istio on Primary cluster: ${CTX_CLUSTER1}" + snip_install_istio_17 + echo y | snip_install_istio_18 - # Expose istiod via east-west gateway. - snip_install_istio_19 - snip_install_istio_20 + echo "Creating the east-west gateway" + snip_install_istio_19 + + echo "Waiting for the east-west gateway to have an external IP" + _wait_for_gateway_ip istio-system istio-eastwestgateway "${CTX_CLUSTER1}" + + echo "Exposing istiod via the east-west gateway" + snip_install_istio_20 } function install_istio_on_cluster2 { - snip_install_istio_21 - snip_install_istio_22 - echo y | snip_install_istio_23 + echo "Installing Istio on Remote cluster: ${CTX_CLUSTER2}" + snip_install_istio_22 + snip_install_istio_23 + echo y | snip_install_istio_24 } -# Install Istio on the 2 clusters. Executing in -# parallel to reduce test time. -install_istio_on_cluster1 & -install_istio_on_cluster2 & -wait +function configure_api_server_access { + snip_install_istio_21 +} -# Configure endpoint discovery. -snip_install_istio_24 - -# Verify that traffic is properly load balanced. -verify_load_balancing +time configure_trust +time install_istio_on_cluster1 +time configure_api_server_access +time install_istio_on_cluster2 +time verify_load_balancing # @cleanup source content/en/docs/setup/install/multicluster/common.sh set +e # ignore cleanup errors set_single_network_vars -cleanup +time cleanup + +# Everything should be removed once cleanup completes. Use a small +# number of retries for comparing cluster snapshots before/after the test. +export VERIFY_RETRIES=1 diff --git a/content/en/docs/setup/install/multicluster/snips.sh b/content/en/docs/setup/install/multicluster/snips.sh index 7479bbd4ca..c0fccf6d43 100644 --- a/content/en/docs/setup/install/multicluster/snips.sh +++ b/content/en/docs/setup/install/multicluster/snips.sh @@ -26,7 +26,7 @@ export CTX_CLUSTER2=cluster2 } snip_install_istio_1() { -cat < ./cluster1.yaml +cat < cluster1.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -44,7 +44,7 @@ istioctl install --context="${CTX_CLUSTER1}" -f cluster1.yaml } snip_install_istio_3() { -cat < ./cluster2.yaml +cat < cluster2.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -76,7 +76,7 @@ istioctl x create-remote-secret \ } snip_install_istio_7() { -cat < ./cluster1.yaml +cat < cluster1.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -106,7 +106,7 @@ kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \ } snip_install_istio_11() { -cat < ./cluster2.yaml +cat < cluster2.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -150,7 +150,7 @@ istioctl x create-remote-secret \ } snip_install_istio_17() { -cat < ./cluster1.yaml +cat < cluster1.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -180,14 +180,21 @@ kubectl apply --context="${CTX_CLUSTER1}" -f \ } snip_install_istio_21() { +istioctl x create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + --name=cluster2 | \ + kubectl apply -f - --context="${CTX_CLUSTER1}" +} + +snip_install_istio_22() { export DISCOVERY_ADDRESS=$(kubectl \ --context="${CTX_CLUSTER1}" \ -n istio-system get svc istio-eastwestgateway \ -o jsonpath='{.status.loadBalancer.ingress[0].ip}') } -snip_install_istio_22() { -cat < ./cluster2.yaml +snip_install_istio_23() { +cat < cluster2.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -201,19 +208,12 @@ spec: EOF } -snip_install_istio_23() { +snip_install_istio_24() { istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml } -snip_install_istio_24() { -istioctl x create-remote-secret \ - --context="${CTX_CLUSTER2}" \ - --name=cluster2 | \ - kubectl apply -f - --context="${CTX_CLUSTER1}" -} - snip_install_istio_25() { -cat < ./cluster1.yaml +cat < cluster1.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -248,14 +248,21 @@ kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \ } snip_install_istio_30() { +istioctl x create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + --name=cluster2 | \ + kubectl apply -f - --context="${CTX_CLUSTER1}" +} + +snip_install_istio_31() { export DISCOVERY_ADDRESS=$(kubectl \ --context="${CTX_CLUSTER1}" \ -n istio-system get svc istio-eastwestgateway \ -o jsonpath='{.status.loadBalancer.ingress[0].ip}') } -snip_install_istio_31() { -cat < ./cluster2.yaml +snip_install_istio_32() { +cat < cluster2.yaml apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: @@ -269,29 +276,22 @@ spec: EOF } -snip_install_istio_32() { +snip_install_istio_33() { istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml } -snip_install_istio_33() { +snip_install_istio_34() { MESH=mesh1 CLUSTER=cluster2 NETWORK=network2 \ samples/multicluster/gen-eastwest-gateway.sh | \ istioctl manifest generate -f - | \ kubectl apply --context="${CTX_CLUSTER2}" -f - } -snip_install_istio_34() { +snip_install_istio_35() { kubectl --context="${CTX_CLUSTER2}" apply -n istio-system -f \ samples/multicluster/expose-services.yaml } -snip_install_istio_35() { -istioctl x create-remote-secret \ - --context="${CTX_CLUSTER2}" \ - --name=cluster2 | \ - kubectl apply -f - --context="${CTX_CLUSTER1}" -} - snip_deploy_the_helloworld_service_1() { kubectl create --context="${CTX_CLUSTER1}" namespace sample kubectl create --context="${CTX_CLUSTER2}" namespace sample @@ -307,20 +307,20 @@ kubectl label --context="${CTX_CLUSTER2}" namespace sample \ snip_deploy_the_helloworld_service_3() { kubectl apply --context="${CTX_CLUSTER1}" \ -f samples/helloworld/helloworld.yaml \ - -l app=helloworld -n sample + -l service=helloworld -n sample kubectl apply --context="${CTX_CLUSTER2}" \ -f samples/helloworld/helloworld.yaml \ - -l app=helloworld -n sample + -l service=helloworld -n sample } snip_deploy_helloworld_v1_1() { kubectl apply --context="${CTX_CLUSTER1}" \ -f samples/helloworld/helloworld.yaml \ - -l app=helloworld -l version=v1 -n sample + -l version=v1 -n sample } snip_deploy_helloworld_v1_2() { -kubectl get pod --context="${CTX_CLUSTER1}" -n sample +kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l app=helloworld } ! read -r -d '' snip_deploy_helloworld_v1_2_out <<\ENDSNIP @@ -331,11 +331,11 @@ ENDSNIP snip_deploy_helloworld_v2_1() { kubectl apply --context="${CTX_CLUSTER2}" \ -f samples/helloworld/helloworld.yaml \ - -l app=helloworld -l version=v2 -n sample + -l version=v2 -n sample } snip_deploy_helloworld_v2_2() { -kubectl get pod --context="${CTX_CLUSTER2}" -n sample +kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l app=helloworld } ! read -r -d '' snip_deploy_helloworld_v2_2_out <<\ENDSNIP @@ -355,6 +355,7 @@ kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l app=sleep } ! read -r -d '' snip_deploy_sleep_2_out <<\ENDSNIP +NAME READY STATUS RESTARTS AGE sleep-754684654f-n6bzf 2/2 Running 0 5s ENDSNIP @@ -363,6 +364,7 @@ kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l app=sleep } ! read -r -d '' snip_deploy_sleep_3_out <<\ENDSNIP +NAME READY STATUS RESTARTS AGE sleep-754684654f-dzl9j 2/2 Running 0 5s ENDSNIP diff --git a/go.mod b/go.mod index aba222fab5..65faafa5d1 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,6 @@ replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.5 replace github.com/evanphx/json-patch => github.com/evanphx/json-patch v0.0.0-20190815234213-e83c0a1c26c8 require ( - istio.io/istio v0.0.0-20201014161623-394493e824eb + istio.io/istio v0.0.0-20201020214016-6ba5c6b9a1e7 istio.io/pkg v0.0.0-20200922180714-670b76a68558 ) diff --git a/go.sum b/go.sum index 8a3b8766d1..3f7cd81076 100644 --- a/go.sum +++ b/go.sum @@ -1233,15 +1233,15 @@ honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= istio.io/api v0.0.0-20190515205759-982e5c3888c6/go.mod h1:hhLFQmpHia8zgaM37vb2ml9iS5NfNfqZGRt1pS9aVEo= istio.io/api v0.0.0-20200812202721-24be265d41c3/go.mod h1:88HN3o1fSD1jo+Z1WTLlJfMm9biopur6Ct9BFKjiB64= -istio.io/api v0.0.0-20201007205536-38d3c76a5557 h1:EDgHyV3kq988v2vb4p4j/D0saXf1JGh43uWhEkD4x+M= -istio.io/api v0.0.0-20201007205536-38d3c76a5557/go.mod h1:88HN3o1fSD1jo+Z1WTLlJfMm9biopur6Ct9BFKjiB64= +istio.io/api v0.0.0-20201020011337-0d3a960deddb h1:/oFtAb7roeWXlZh9CDWM4pQsPeDfunzxVg4aldSLw84= +istio.io/api v0.0.0-20201020011337-0d3a960deddb/go.mod h1:88HN3o1fSD1jo+Z1WTLlJfMm9biopur6Ct9BFKjiB64= istio.io/client-go v0.0.0-20200908160912-f99162621a1a h1:clPn0fz+rXq5Ytj6Ppb1ygUKeU0RImT4ZbT1oMd1G04= istio.io/client-go v0.0.0-20200908160912-f99162621a1a/go.mod h1:SO65MWt7I45dvUwuDowoiB0SVcGpfWZfUTlopvYpbZc= istio.io/gogo-genproto v0.0.0-20190930162913-45029607206a/go.mod h1:OzpAts7jljZceG4Vqi5/zXy/pOg1b209T3jb7Nv5wIs= istio.io/gogo-genproto v0.0.0-20200908160912-66171252e3db h1:btPv5pCusFxbWrmiLNLEThH8IzWunUeZ1r5M1O22vBc= istio.io/gogo-genproto v0.0.0-20200908160912-66171252e3db/go.mod h1:OzpAts7jljZceG4Vqi5/zXy/pOg1b209T3jb7Nv5wIs= -istio.io/istio v0.0.0-20201014161623-394493e824eb h1:TtFRcM8eYDPvZOArA1Ykne9JMzXSPMPBOqVnoT/oUUM= -istio.io/istio v0.0.0-20201014161623-394493e824eb/go.mod h1:bOY51cI9r95WV7gSP4r/bWzaxcCjaXuNGBvLEJQ0wJc= +istio.io/istio v0.0.0-20201020214016-6ba5c6b9a1e7 h1:6CJDE2X7mH6pcNpNNniD8r9ZSnDZn7xo6Ay3VewfkJ0= +istio.io/istio v0.0.0-20201020214016-6ba5c6b9a1e7/go.mod h1:nPwhi1qIogLf4pxXfdh86h+A4TM4SU5JfxSsvroaXIQ= istio.io/pkg v0.0.0-20200922180714-670b76a68558 h1:ATDshla3gX4b0X8oMs0rrFry9qJshrzTnFkBV42CcFk= istio.io/pkg v0.0.0-20200922180714-670b76a68558/go.mod h1:p6wktGBjkjL3spRSsyfOh0XkuKb8IuBX61rERHfmSbU= k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= @@ -1250,10 +1250,10 @@ k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= k8s.io/api v0.18.3 h1:2AJaUQdgUZLoDZHrun21PW2Nx9+ll6cUzvn3IKhSIn0= k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA= k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= -k8s.io/api v0.19.0 h1:XyrFIJqTYZJ2DU7FBE/bSPz7b1HvbVBuBf07oeo6eTc= -k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= k8s.io/api v0.19.1 h1:oZf4bYsBdjC49PdTwNfLmrfUFCwKUi94HY/+emXI8Qw= k8s.io/api v0.19.1/go.mod h1:+u/k4/K/7vp4vsfdT7dyl8Oxk1F26Md4g5F26Tu85PU= +k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= +k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= @@ -1265,10 +1265,10 @@ k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftc k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk= k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apimachinery v0.19.0 h1:gjKnAda/HZp5k4xQYjL0K/Yb66IvNqjthCb03QlKpaQ= -k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.19.1 h1:cwsxZazM/LA9aUsBaL4bRS5ygoM6bYp8dFk22DSYQa4= k8s.io/apimachinery v0.19.1/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc= +k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= @@ -1280,15 +1280,16 @@ k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= k8s.io/client-go v0.18.1/go.mod h1:iCikYRiXOj/yRRFE/aWqrpPtDt4P2JVWhtHkmESTcfY= k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= -k8s.io/client-go v0.19.0 h1:1+0E0zfWFIWeyRhQYWzimJOyAk2UT7TiARaLNwJCf7k= -k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= k8s.io/client-go v0.19.1 h1:xfFwj+YFKa8rcihlFYZABjxcy7Sm/wJQ+GxW3JyVtKI= k8s.io/client-go v0.19.1/go.mod h1:AZOIVSI9UUtQPeJD3zJFp15CEhSjRgAuQP5PWRJrCIQ= +k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= +k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= k8s.io/code-generator v0.19.1/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= @@ -1333,8 +1334,8 @@ sigs.k8s.io/controller-runtime v0.6.2/go.mod h1:vhcq/rlnENJ09SIRp3EveTaZ0yqH526h sigs.k8s.io/controller-tools v0.4.0/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= -sigs.k8s.io/service-apis v0.0.0-20200916220245-b060b8df63c9 h1:9z7RSE6mFtwzhHvuZWqki6KSLp9g94jt17ysvft/nEU= -sigs.k8s.io/service-apis v0.0.0-20200916220245-b060b8df63c9/go.mod h1:yfIjKFwmoJxpP4m4qOmdBLgJvlSw61GDYuKTuRo7O1c= +sigs.k8s.io/service-apis v0.0.0-20201013190827-799dbbe3a3ba h1:tmIsMOyPg1eOSRDFMinrHktWafHyosZcxK/hnFS5HFo= +sigs.k8s.io/service-apis v0.0.0-20201013190827-799dbbe3a3ba/go.mod h1:QkiV/PnK7YbN5zqYqXnh5wByTTT1LYJ5scwdIs62qWs= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/tests/util/helpers.sh b/tests/util/helpers.sh index 65b1b69a6b..1e51efeb53 100644 --- a/tests/util/helpers.sh +++ b/tests/util/helpers.sh @@ -70,16 +70,50 @@ _set_ingress_environment_variables() { # kubectl wait --for=condition=Ready pod --all --timeout=60s # Wait for rollout of named deployment -# usage: _wait_for_deployment +# usage: _wait_for_deployment _wait_for_deployment() { local namespace="$1" local name="$2" - if ! kubectl -n "$namespace" rollout status deployment "$name" --timeout 5m; then + local context="${3:-}" + if ! kubectl --context="$context" -n "$namespace" rollout status deployment "$name" --timeout 5m; then echo "Failed rollout of deployment $name in namespace $namespace" exit 1 fi } +# Wait for the given gateway to be allocated an external IP. +# usage: _wait_for_gateway_ip +_wait_for_gateway_ip() { + local namespace="$1" + local service="$2" + local context="${3:-}" + + local max_time=${MAX_SECONDS:-300} # Default to 5 min. + local delay=5 + + local start_time=$(date +%s) + local current_time=$start_time + local end_time=$((start_time + max_time)) + + while true; do + local ip=$(kubectl --context="${context}" get svc "${service}" -n "${namespace}" -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + + # Verify that the IP is set. + if [[ -n "${ip}" ]]; then + echo "IP Assigned for $service.$namespace: ${ip}" + return + fi + + current_time=$(date +%s) + if (( current_time > end_time )); then + echo "Failed waiting for $service.$namespace: ${ip}" + exit 1 + fi + + sleep "${delay}" + done +} + # Wait for Istio config to propagate # usage: _wait_for_istio _wait_for_istio() { diff --git a/tests/util/verify.sh b/tests/util/verify.sh index 87568c8595..2b8b920a97 100644 --- a/tests/util/verify.sh +++ b/tests/util/verify.sh @@ -313,6 +313,8 @@ __create_cluster_snapshots() { } __cluster_cleanup_check() { + VERIFY_RETRIES=${VERIFY_RETRIES:-9} + # Get the list of KUBECONFIG files as an array. IFS=':' read -r -a KFILES <<< "${KUBECONFIG}" for KFILE in "${KFILES[@]}"; do @@ -330,7 +332,6 @@ __cluster_cleanup_check() { rm "${SNAPSHOT_FILE}" # Verify that we've restored the original cluster state. - VERIFY_RETRIES=9 (KUBECONFIG="${KFILE}"; _verify_like __cluster_state "${SNAPSHOT}") echo "Finished cleanup check against snapshot ${SNAPSHOT_FILE}" done