Verify test cleanup (#7889)

* Verify test cleanup

* disable istio-system checks

* more tolerant checks

* fix lint

* more lint

* limited check cleanup

* always check

* snapshot tweak

content/en/docs/tasks/traffic-management/egress/egress-gateway-tls-origination-sds/index.md

@@ -356,7 +356,7 @@ to hold the configuration of the NGINX server:
 1.  Remove the Istio configuration items you created:
 
     {{< text bash >}}
-    $ kubectl delete destinationrule originate-tls-for-nginx
+    $ kubectl delete destinationrule originate-tls-for-nginx -n istio-system
     $ kubectl delete virtualservice direct-nginx-through-egress-gateway
     $ kubectl delete destinationrule egressgateway-for-nginx
     $ kubectl delete gateway istio-egressgateway

content/en/docs/tasks/traffic-management/egress/egress-gateway-tls-origination-sds/mtls_test.sh

@@ -27,10 +27,10 @@ set -o pipefail
 # TODO: above command is not needed, since access logging seems to be enabled by default.
 # TODO: Also, running "istioctl install" causes the test to fail?????
 
-# Deploy sleep sample and set up variable pointing to it
-set +e
-kubectl delete pods -l app=sleep --force
-set -e
+# Make sure automatic sidecar injection is enabled
+kubectl label namespace default istio-injection=enabled || true
+
+# Deploy sleep sample
 snip_before_you_begin_1
 _wait_for_deployment default sleep
 

content/en/docs/tasks/traffic-management/egress/egress-gateway-tls-origination-sds/snips.sh

@@ -263,7 +263,7 @@ kubectl logs -l istio=egressgateway -n istio-system | grep 'my-nginx.mesh-extern
 }
 
 snip_cleanup_the_tls_origination_example_1() {
-kubectl delete destinationrule originate-tls-for-nginx
+kubectl delete destinationrule originate-tls-for-nginx -n istio-system
 kubectl delete virtualservice direct-nginx-through-egress-gateway
 kubectl delete destinationrule egressgateway-for-nginx
 kubectl delete gateway istio-egressgateway

content/en/docs/tasks/traffic-management/egress/egress-gateway-tls-origination-sds/tls_test.sh

@@ -21,16 +21,10 @@ set -e
 set -u
 set -o pipefail
 
-# Enable access logging
-# istioctl install --set profile=demo --set meshConfig.accessLogFile="/dev/stdout"
-# _wait_for_deployment istio-system istiod
-# TODO: above command is not needed, since access logging seems to be enabled by default.
-# TODO: Also, running "istioctl install" causes the test to fail?????
+# Make sure automatic sidecar injection is enabled
+kubectl label namespace default istio-injection=enabled || true
 
-# Deploy sleep sample and set up variable pointing to it
-set +e
-kubectl delete pods -l app=sleep --force
-set -e
+# Deploy sleep sample
 snip_before_you_begin_1
 _wait_for_deployment default sleep
 
@@ -75,3 +69,4 @@ set +e # ignore cleanup errors
 snip_cleanup_the_tls_origination_example_1
 snip_cleanup_the_tls_origination_example_2
 snip_cleanup_the_tls_origination_example_3
+snip_cleanup_1

content/en/docs/tasks/traffic-management/egress/egress-gateway-tls-origination/mtls_test.sh

@@ -21,6 +21,13 @@ set -e
 set -u
 set -o pipefail
 
+# Make sure automatic sidecar injection is enabled
+kubectl label namespace default istio-injection=enabled || true
+
+# Deploy sleep sample
+snip_before_you_begin_1
+_wait_for_deployment default sleep
+
 # Generate Certificates for service outside the mesh to use for mTLS
 set +e # suppress harmless "No such file or directory:../crypto/bio/bss_file.c:72:fopen('1_root/index.txt.attr','r')" error
 snip_generate_client_and_server_certificates_and_keys_1
@@ -40,14 +47,6 @@ snip_deploy_a_mutual_tls_server_5
 # Wait for nginx
 _wait_for_deployment mesh-external my-nginx
 
-# Deploy Sleep example
-set +e
-kubectl delete pods -l app=sleep --force
-set -e
-snip_before_you_begin_1
-
-_wait_for_deployment default sleep
-
 # Mount certs to Egress Gateway
 snip_redeploy_the_egress_gateway_with_the_client_certificates_1
 snip_redeploy_the_egress_gateway_with_the_client_certificates_2

content/en/docs/tasks/traffic-management/egress/egress-gateway-tls-origination/tls_test.sh

@@ -21,17 +21,11 @@ set -e
 set -u
 set -o pipefail
 
-# Enable access logging
-#istioctl install --set profile=demo --set meshConfig.accessLogFile="/dev/stdout"
-#_wait_for_deployment istio-system istiod
-# TODO: above command is not needed, since access logging seems to be enabled by default.
-# TODO: Also, running "istioctl install" causes the test to fail?????
+# Make sure automatic sidecar injection is enabled
+kubectl label namespace default istio-injection=enabled || true
 
 # Deploy sleep sample and set up variable pointing to it
-set +e
-kubectl delete pods -l app=sleep --force
-set -e
-snip_before_you_begin_2
+snip_before_you_begin_1
 _wait_for_deployment default sleep
 snip_before_you_begin_3
 
@@ -58,3 +52,4 @@ _verify_contains snip_perform_tls_origination_with_an_egress_gateway_6 "GET /pol
 # @cleanup
 set +e # ignore cleanup errors
 snip_cleanup_the_tls_origination_example_1
+snip_cleanup_1

content/en/docs/tasks/traffic-management/egress/egress-gateway/test.sh

@@ -49,9 +49,7 @@ _verify_contains snip_egress_gateway_for_http_traffic_5 "HTTP/2 200"
 _verify_contains snip_egress_gateway_for_http_traffic_6 "GET /politics HTTP/2"
 
 # cleanup http task
-set +e # ignore cleanup errors
 snip_cleanup_http_gateway_1
-set -e
 
 # ServiceEntry for HTTPS
 snip_egress_gateway_for_https_traffic_1
@@ -72,9 +70,7 @@ _verify_contains snip_egress_gateway_for_https_traffic_4 "HTTP/2 200"
 _verify_contains snip_egress_gateway_for_https_traffic_5 "outbound|443||edition.cnn.com"
 
 # cleanup https
-set +e # ignore cleanup errors
 snip_cleanup_https_gateway_1
-set -e
 
 ### Kubernetes netowkring policy test
 
@@ -119,5 +115,7 @@ _verify_contains snip_apply_kubernetes_network_policies_14 "outbound|443||editio
 
 # @cleanup
 set +e # ignore cleanup errors
+snip_cleanup_http_gateway_1
+snip_cleanup_https_gateway_1
 snip_cleanup_network_policies_1
 snip_cleanup_1

content/en/docs/tasks/traffic-management/tcp-traffic-shifting/test.sh

@@ -21,17 +21,7 @@ set -o pipefail
 
 source "tests/util/samples.sh"
 
-# @setup profile=default
-
-# TODO: why is the following needed in the test if it's not a needed step in the doc?
-# add the TCP port to the ingress-gateway
-kubectl -n istio-system patch service istio-ingressgateway --patch "
-spec:
-  ports:
-    - port: 31400
-      targetPort: 31400
-      name: tcp
-"
+# @setup profile=demo
 
 # create a new namespace for testing purposes and enable automatic Istio sidecar injection
 snip_set_up_the_test_environment_1
@@ -78,4 +68,3 @@ _verify_lines snip_apply_weightbased_tcp_routing_5 "
 # @cleanup
 set +e # ignore cleanup errors
 snip_cleanup_1
-cleanup_sleep_sample

pkg/test/istioio/framework.go

@@ -56,6 +56,14 @@ var (
 		source "tests/util/helpers.sh"
 	`
 
+	clusterSnapshot = `
+		__cluster_snapshot
+	`
+
+	clusterCleanupCheck = `
+		__cluster_cleanup_check
+	`
+
 	snipsFileSuffix = "snips.sh"
 	testFileSuffix  = "test.sh"
 
@@ -146,6 +154,10 @@ func checkFile(path string) (*TestCase, error) {
 	}
 	config := setups[0][1]
 
+	// Check for proper test cleanup
+	testScript = clusterSnapshot + testScript
+	cleanupScript += clusterCleanupCheck
+
 	testCase = &TestCase{
 		valid:         true,
 		path:          shortPath,

tests/util/verify.sh

@@ -258,6 +258,51 @@ __verify_with_retry() {
     done
 }
 
+# Get the resource state of the cluster. Used by the test framework to compare the
+# cluster state before and after running each test:
+#
+# __cluster_snapshot
+#
+# ... test commands
+# ... cleanup commands
+#
+# __cluster_cleanup_check
+#
+__cluster_state() {
+    # kubectl get ns -o name
+    # kubectl get all --ignore-not-found -n default -n istio-system
+    # kubectl get istiooperators --ignore-not-found -n default -n istio-system
+    # TODO: ^^^ fails because istio-system ns is sometimes incorrectly in snapshot, still cleaning up from previous test.
+
+    # TEMP WORKAROUND, don't check istio-system
+    kubectl get ns -o name | sed '/istio-system/d'
+    kubectl get all --ignore-not-found -n default
+    kubectl get istiooperators --ignore-not-found -n default
+
+    kubectl get destinationrules --ignore-not-found -n default -n istio-system
+    kubectl get envoyfilters --ignore-not-found -n default -n istio-system
+    kubectl get gateways --ignore-not-found -n default -n istio-system
+    kubectl get serviceentries --ignore-not-found -n default -n istio-system
+    kubectl get sidecars --ignore-not-found -n default -n istio-system
+    kubectl get virtualservices --ignore-not-found -n default -n istio-system
+    kubectl get workloadentries --ignore-not-found -n default -n istio-system
+    kubectl get authorizationpolicies --ignore-not-found -n default -n istio-system
+    kubectl get peerauthentications --ignore-not-found -n default -n istio-system
+    kubectl get requestauthentications --ignore-not-found -n default -n istio-system
+}
+
+__cluster_snapshot() {
+    __cluster_state > __cluster_snapshot.txt 2>&1
+}
+
+__cluster_cleanup_check() {
+    snapshot=$(<__cluster_snapshot.txt)
+    rm __cluster_snapshot.txt
+
+    VERIFY_RETRIES=10
+    _verify_like __cluster_state "$snapshot"
+}
+
 
 # Public Functions