Add built-in verify functions to test framework (#7110)

* Add builtin verify functions to test framework

* fix lint

* tweaks

* fix bold

pkg/test/istioio/script.go

@@ -300,7 +300,7 @@ func (s Script) runCommand(ctx Context) {
 	}
 
 	// Generate the body of the command.
-	commandLines := make([]string, 0)
+	commandLines := []string{"source ${REPO_ROOT}/tests/util/verify.sh"}
 	lines := strings.Split(content, "\n")
 	for index := 0; index < len(lines); index++ {
 		line := lines[index]

tests/README.md

@@ -116,36 +116,40 @@ Your test script can then invoke the commands by simply calling snip functions:
 snip_config_50_v3 # Step 3: switch 50% traffic to v3
 ```
 
-To verify the output, you currently have two choices:
-
-1. Surround the call with `# $snippet` and `# $endsnippet` comments,
-    including `# $verify` followed by the expected output:
-
-    ```sh
-    # $snippet
-    snip_config_50_v3
-    # $verify
-    virtualservice.networking.istio.io/reviews configured
-    # $endsnippet
-    ```
-
-    **Note**: There should be no other fields on the line following the `# $snippet` directive.
-    The `# $snippet`, without a following name, will simply run and verify the commands
-    in the snippet section, i.e., no output snippet will be generated.
-
-    Refer to [verifier.go](../pkg/test/istioio/verifier.go) for supported verifiers.
-
-1. Capture the command output and compare it to a variable containing the expected output:
+For commands that produce output that needs to be verified, capture the command output
+in a variable and compare it to the expected output. For example:
 
 ```sh
 out=$(snip_set_up_the_cluster_3 2>&1)
-    if [ "$out" != "$snip_set_up_the_cluster_3_out" ]; then
-        echo "FAILED snip_set_up_the_cluster_3: $out"; exit 1
-    fi
+_verify_same "$out" "$snip_set_up_the_cluster_3_out" "snip_set_up_the_cluster_3"
 ```
 
-    TODO: Add built-in verifier functions that can be used instead of simple string compare.
-    Once this is available, we can deprecate the `# $verify` approach.
+The framework includes the following built-in verify functions:
+
+1. **`_verify_same`** `out` `expected` `msg`
+
+   Verify that `out` is exactly the same as `expected`. Failure messages will include
+   the specified `msg`.
+
+1. **`_verify_contains`** `out` `expected` `msg`
+
+   Verify that `out` contains the substring `expected`. Failure messages will include
+   the specified `msg`.
+
+1. **`_verify_like`** `out` `expected` `msg`
+
+   Verify that `out` is "like" `expected`. Like implies:
+
+   - Same number of lines
+   - Same number of whitespace-seperated tokens per line
+   - Tokens can only differ in the following ways:
+
+     1. different elapsed time values (e.g., `30s`)
+     1. different ip values (e.g., `172.21.0.1`)
+     1. prefix match ending with a dash character (e.g., `reviews-v1-<anything>`)
+
+   This function is useful for comparing the output of commands that include some run-specific
+   values in the output (e.g., `kubectl get pods`), or when whitespace in the output may be different.
 
 ## Builder
 

tests/security/plugin_ca_cert/scripts/plugin_ca_cert.txt

@@ -35,16 +35,10 @@ set -o pipefail
 snip_verifying_the_certificates_2
 
 out=$(snip_verifying_the_certificates_3 2>&1)
-if [ "$out" != "$snip_verifying_the_certificates_3_out" ]; then
-    echo "FAILED snip_verifying_the_certificates_3: $out"; exit 1
-fi
+_verify_same "$out" "$snip_verifying_the_certificates_3_out" "snip_verifying_the_certificates_3"
 
 out=$(snip_verifying_the_certificates_4 2>&1)
-if [ "$out" != "$snip_verifying_the_certificates_4_out" ]; then
-    echo "FAILED snip_verifying_the_certificates_4: $out"; exit 1
-fi
+_verify_same "$out" "$snip_verifying_the_certificates_4_out" "snip_verifying_the_certificates_4"
 
 out=$(snip_verifying_the_certificates_5 2>&1)
-if [ "$out" != "$snip_verifying_the_certificates_5_out" ]; then
-    echo "FAILED snip_verifying_the_certificates_5: $out"; exit 1
-fi
+_verify_same "$out" "$snip_verifying_the_certificates_5_out" "snip_verifying_the_certificates_5"

tests/security/scripts/mtls_migration.txt

@@ -22,21 +22,15 @@ source ${REPO_ROOT}/content/en/docs/tasks/security/authentication/mtls-migration
 
 # curl_foo_bar_legacy
 out=$(snip_set_up_the_cluster_3 2>&1)
-if [ "$out" != "$snip_set_up_the_cluster_3_out" ]; then
-    echo "FAILED snip_set_up_the_cluster_3: $out"; exit 1
-fi
+_verify_same "$out" "$snip_set_up_the_cluster_3_out" "snip_set_up_the_cluster_3"
 
 # verify_initial_peerauthentications
 out=$(snip_set_up_the_cluster_4 2>&1)
-if [ "$out" != "$snip_set_up_the_cluster_4_out" ]; then
-    echo "FAILED snip_set_up_the_cluster_4: $out"; exit 1
-fi
+_verify_same "$out" "$snip_set_up_the_cluster_4_out" "snip_set_up_the_cluster_4"
 
 # verify_initial_destinationrules
-out=$(snip_set_up_the_cluster_4 2>&1)
-if [ "$out" != "$snip_set_up_the_cluster_4_out" ]; then
-    echo "FAILED snip_set_up_the_cluster_4: $out"; exit 1
-fi
+out=$(snip_set_up_the_cluster_5 2>&1)
+_verify_same "$out" "$snip_set_up_the_cluster_5_out" "snip_set_up_the_cluster_5"
 
 # configure_mtls_foo_peerauthentication
 snip_lock_down_to_mutual_tls_by_namespace_1
@@ -47,9 +41,7 @@ set +o pipefail
 
 # curl_foo_bar_legacy_post_pa
 out=$(snip_lock_down_to_mutual_tls_by_namespace_2 2>&1)
-if [ "$out" != "$snip_lock_down_to_mutual_tls_by_namespace_2_out" ]; then
-    echo "FAILED snip_lock_down_to_mutual_tls_by_namespace_2: $out"; exit 1
-fi
+_verify_same "$out" "$snip_lock_down_to_mutual_tls_by_namespace_2_out" "snip_lock_down_to_mutual_tls_by_namespace_2"
 
 # Restore error handling
 set -e
@@ -63,20 +55,16 @@ set +e
 set +o pipefail
 
 # curl_foo_bar_legacy_httpbin_foo_mtls
-! read -r -d '' expected <<ENDSNIP # No expected output in doc, so declare it here.
-sleep.foo to httpbin.foo: 200
+out=$(snip_lock_down_mutual_tls_for_the_entire_mesh_2 2>&1)
+expected="sleep.foo to httpbin.foo: 200
 sleep.foo to httpbin.bar: 200
 sleep.bar to httpbin.foo: 200
 sleep.bar to httpbin.bar: 200
 sleep.legacy to httpbin.foo: 000
 command terminated with exit code 56
 sleep.legacy to httpbin.bar: 000
-command terminated with exit code 56
-ENDSNIP
-out=$(snip_lock_down_mutual_tls_for_the_entire_mesh_2 2>&1)
-if [ "$out" != "$expected" ]; then
-    echo "FAILED snip_lock_down_mutual_tls_for_the_entire_mesh_2: $out"; exit 1
-fi
+command terminated with exit code 56"
+_verify_same "$out" "$expected" "snip_lock_down_mutual_tls_for_the_entire_mesh_2"
 
 # Restore error handling
 set -e

tests/trafficmanagement/scripts/traffic_shifting.txt

@@ -78,45 +78,37 @@ function verify_traffic_shift() {
   fi
 }
 
-# Step 1
+# Step 1 configure all traffic to v1
 
-# $snippet
-snip_config_all_v1
-# $verify
-virtualservice.networking.istio.io/productpage created
+out=$(snip_config_all_v1 2>&1)
+expected="virtualservice.networking.istio.io/productpage created
 virtualservice.networking.istio.io/reviews created
 virtualservice.networking.istio.io/ratings created
-virtualservice.networking.istio.io/details created
-# $endsnippet
+virtualservice.networking.istio.io/details created"
+_verify_same "$out" "$expected" "snip_config_all_v1"
 
 # Step 2: verify no rating stars visible, (reviews-v3 traffic=0%)
+
 verify_traffic_shift 0
 
 # Step 3: switch 50% traffic to v3
 
-# $snippet
-snip_config_50_v3
-# $verify
-virtualservice.networking.istio.io/reviews configured
-# $endsnippet
+out=$(snip_config_50_v3 2>&1)
+_verify_same "$out" "virtualservice.networking.istio.io/reviews configured" "snip_config_50_v3"
 
 istioctl experimental wait --for=distribution VirtualService reviews.default
 
 # Step 4: Confirm the rule was replaced
 
-# $snippet
-snip_verify_config_50_v3
-# $verify verifier="contains"
-subset: v3
-# $endsnippet
+out=$(snip_verify_config_50_v3 2>&1)
+_verify_contains "$out" "subset: v3" "snip_verify_config_50_v3"
 
 # Step 5: verify rating stars visible 50% of the time
+
 verify_traffic_shift 50
 
 # Step 6: route 100% traffic to v3
 
-# $snippet
 snip_config_100_v3
-# $endsnippet
 
 verify_traffic_shift 100

tests/util/verify.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+_err_exit() {
+    echo "VERIFY FAILED $1: $2";
+    exit 1
+}
+
+# Verify that $out is the same as $expected.
+_verify_same() {
+    local out=$1
+    local expected=$2
+    local msg=$3
+    if [ "$out" != "$expected" ]; then
+        _err_exit "$msg" "$out"
+    fi
+}
+
+# Verify that $out contains the substring $expected.
+_verify_contains() {
+    local out=$1
+    local expected=$2
+    local msg=$3
+    
+    if [[ "$out" != *"$expected"* ]]; then
+        _err_exit "$msg" "$out"
+    fi
+}
+
+# Verify that $out is "like" $expected. Like implies:
+#   1. Same number of lines
+#   2. Same number of whitespace-seperated tokens per line
+#   3. Tokens can only differ in the following ways:
+#        - different elapsed time values
+#        - different ip values
+#        - prefix match ending with a dash character
+_verify_like() {
+    local out=$1
+    local expected=$2
+    local msg=$3
+    
+    if [[ "$out" != "$expected" ]]; then
+        local olines=()
+        while read -r line; do
+            olines+=("$line")
+        done <<< "$out"
+
+        local elines=()
+        while read -r line; do
+            elines+=("$line")
+        done <<< "$expected"
+
+        if [[ ${#olines[@]} -ne ${#elines[@]} ]]; then
+            _err_exit "$msg" "$out"
+        fi
+
+        for i in "${!olines[@]}"; do
+            local oline=${olines[i]}
+            local eline=${elines[i]}
+
+            if [[ "$oline" == "$eline" ]]; then
+                continue
+            fi
+
+            read -r -a otokens <<< "$oline"
+            read -r -a etokens <<< "$eline"
+            
+            if [[ ${#otokens[@]} -ne ${#etokens[@]} ]]; then
+                _err_exit "$msg" "$out"
+            fi
+
+            for j in "${!otokens[@]}"; do
+                local otok=${otokens[j]}
+                local etok=${etokens[j]}
+
+                if [[ "$otok" == "$etok" ]]; then
+                    continue
+                fi
+
+                if [[ "$otok" =~ ^[0-9]+[smhd]$ && "$etok" =~ ^[0-9]+[smhd]$ ]]; then
+                    continue
+                fi
+
+                if [[ "$otok" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ && "$etok" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+                    continue
+                fi
+
+                local comm=""
+                for ((k=0; k < ${#otok}; k++)) do
+                    if [ "${otok:$k:1}" = "${etok:$k:1}" ]; then
+                        comm=${comm}${otok:$k:1}
+                    else
+                        if [[ "$comm" =~ ^([a-zA-Z0-9_]+-)+ ]]; then
+                            break
+                        fi
+                        _err_exit "$msg" "$out"
+                    fi
+                done
+            done
+        done
+    fi
+}