diff --git a/pkg/test/istioio/script.go b/pkg/test/istioio/script.go index 4fcebad149..da030fa9ca 100644 --- a/pkg/test/istioio/script.go +++ b/pkg/test/istioio/script.go @@ -300,7 +300,7 @@ func (s Script) runCommand(ctx Context) { } // Generate the body of the command. - commandLines := make([]string, 0) + commandLines := []string{"source ${REPO_ROOT}/tests/util/verify.sh"} lines := strings.Split(content, "\n") for index := 0; index < len(lines); index++ { line := lines[index] diff --git a/tests/README.md b/tests/README.md index 7ebbd1d960..7520a121dd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -116,36 +116,40 @@ Your test script can then invoke the commands by simply calling snip functions: snip_config_50_v3 # Step 3: switch 50% traffic to v3 ``` -To verify the output, you currently have two choices: +For commands that produce output that needs to be verified, capture the command output +in a variable and compare it to the expected output. For example: -1. Surround the call with `# $snippet` and `# $endsnippet` comments, - including `# $verify` followed by the expected output: +```sh +out=$(snip_set_up_the_cluster_3 2>&1) +_verify_same "$out" "$snip_set_up_the_cluster_3_out" "snip_set_up_the_cluster_3" +``` - ```sh - # $snippet - snip_config_50_v3 - # $verify - virtualservice.networking.istio.io/reviews configured - # $endsnippet - ``` +The framework includes the following built-in verify functions: - **Note**: There should be no other fields on the line following the `# $snippet` directive. - The `# $snippet`, without a following name, will simply run and verify the commands - in the snippet section, i.e., no output snippet will be generated. +1. **`_verify_same`** `out` `expected` `msg` - Refer to [verifier.go](../pkg/test/istioio/verifier.go) for supported verifiers. + Verify that `out` is exactly the same as `expected`. Failure messages will include + the specified `msg`. -1. Capture the command output and compare it to a variable containing the expected output: +1. **`_verify_contains`** `out` `expected` `msg` - ```sh - out=$(snip_set_up_the_cluster_3 2>&1) - if [ "$out" != "$snip_set_up_the_cluster_3_out" ]; then - echo "FAILED snip_set_up_the_cluster_3: $out"; exit 1 - fi - ``` + Verify that `out` contains the substring `expected`. Failure messages will include + the specified `msg`. - TODO: Add built-in verifier functions that can be used instead of simple string compare. - Once this is available, we can deprecate the `# $verify` approach. +1. **`_verify_like`** `out` `expected` `msg` + + Verify that `out` is "like" `expected`. Like implies: + + - Same number of lines + - Same number of whitespace-seperated tokens per line + - Tokens can only differ in the following ways: + + 1. different elapsed time values (e.g., `30s`) + 1. different ip values (e.g., `172.21.0.1`) + 1. prefix match ending with a dash character (e.g., `reviews-v1-`) + + This function is useful for comparing the output of commands that include some run-specific + values in the output (e.g., `kubectl get pods`), or when whitespace in the output may be different. ## Builder diff --git a/tests/security/plugin_ca_cert/scripts/plugin_ca_cert.txt b/tests/security/plugin_ca_cert/scripts/plugin_ca_cert.txt index aac23ee918..637b304dba 100644 --- a/tests/security/plugin_ca_cert/scripts/plugin_ca_cert.txt +++ b/tests/security/plugin_ca_cert/scripts/plugin_ca_cert.txt @@ -35,16 +35,10 @@ set -o pipefail snip_verifying_the_certificates_2 out=$(snip_verifying_the_certificates_3 2>&1) -if [ "$out" != "$snip_verifying_the_certificates_3_out" ]; then - echo "FAILED snip_verifying_the_certificates_3: $out"; exit 1 -fi +_verify_same "$out" "$snip_verifying_the_certificates_3_out" "snip_verifying_the_certificates_3" out=$(snip_verifying_the_certificates_4 2>&1) -if [ "$out" != "$snip_verifying_the_certificates_4_out" ]; then - echo "FAILED snip_verifying_the_certificates_4: $out"; exit 1 -fi +_verify_same "$out" "$snip_verifying_the_certificates_4_out" "snip_verifying_the_certificates_4" out=$(snip_verifying_the_certificates_5 2>&1) -if [ "$out" != "$snip_verifying_the_certificates_5_out" ]; then - echo "FAILED snip_verifying_the_certificates_5: $out"; exit 1 -fi +_verify_same "$out" "$snip_verifying_the_certificates_5_out" "snip_verifying_the_certificates_5" diff --git a/tests/security/scripts/mtls_migration.txt b/tests/security/scripts/mtls_migration.txt index ff30d772ca..4f39b091a4 100644 --- a/tests/security/scripts/mtls_migration.txt +++ b/tests/security/scripts/mtls_migration.txt @@ -22,21 +22,15 @@ source ${REPO_ROOT}/content/en/docs/tasks/security/authentication/mtls-migration # curl_foo_bar_legacy out=$(snip_set_up_the_cluster_3 2>&1) -if [ "$out" != "$snip_set_up_the_cluster_3_out" ]; then - echo "FAILED snip_set_up_the_cluster_3: $out"; exit 1 -fi +_verify_same "$out" "$snip_set_up_the_cluster_3_out" "snip_set_up_the_cluster_3" # verify_initial_peerauthentications out=$(snip_set_up_the_cluster_4 2>&1) -if [ "$out" != "$snip_set_up_the_cluster_4_out" ]; then - echo "FAILED snip_set_up_the_cluster_4: $out"; exit 1 -fi +_verify_same "$out" "$snip_set_up_the_cluster_4_out" "snip_set_up_the_cluster_4" # verify_initial_destinationrules -out=$(snip_set_up_the_cluster_4 2>&1) -if [ "$out" != "$snip_set_up_the_cluster_4_out" ]; then - echo "FAILED snip_set_up_the_cluster_4: $out"; exit 1 -fi +out=$(snip_set_up_the_cluster_5 2>&1) +_verify_same "$out" "$snip_set_up_the_cluster_5_out" "snip_set_up_the_cluster_5" # configure_mtls_foo_peerauthentication snip_lock_down_to_mutual_tls_by_namespace_1 @@ -47,9 +41,7 @@ set +o pipefail # curl_foo_bar_legacy_post_pa out=$(snip_lock_down_to_mutual_tls_by_namespace_2 2>&1) -if [ "$out" != "$snip_lock_down_to_mutual_tls_by_namespace_2_out" ]; then - echo "FAILED snip_lock_down_to_mutual_tls_by_namespace_2: $out"; exit 1 -fi +_verify_same "$out" "$snip_lock_down_to_mutual_tls_by_namespace_2_out" "snip_lock_down_to_mutual_tls_by_namespace_2" # Restore error handling set -e @@ -63,20 +55,16 @@ set +e set +o pipefail # curl_foo_bar_legacy_httpbin_foo_mtls -! read -r -d '' expected <&1) +expected="sleep.foo to httpbin.foo: 200 sleep.foo to httpbin.bar: 200 sleep.bar to httpbin.foo: 200 sleep.bar to httpbin.bar: 200 sleep.legacy to httpbin.foo: 000 command terminated with exit code 56 sleep.legacy to httpbin.bar: 000 -command terminated with exit code 56 -ENDSNIP -out=$(snip_lock_down_mutual_tls_for_the_entire_mesh_2 2>&1) -if [ "$out" != "$expected" ]; then - echo "FAILED snip_lock_down_mutual_tls_for_the_entire_mesh_2: $out"; exit 1 -fi +command terminated with exit code 56" +_verify_same "$out" "$expected" "snip_lock_down_mutual_tls_for_the_entire_mesh_2" # Restore error handling set -e diff --git a/tests/trafficmanagement/scripts/traffic_shifting.txt b/tests/trafficmanagement/scripts/traffic_shifting.txt index 0c293f8d25..98fcdc9ffd 100644 --- a/tests/trafficmanagement/scripts/traffic_shifting.txt +++ b/tests/trafficmanagement/scripts/traffic_shifting.txt @@ -78,45 +78,37 @@ function verify_traffic_shift() { fi } -# Step 1 +# Step 1 configure all traffic to v1 -# $snippet -snip_config_all_v1 -# $verify -virtualservice.networking.istio.io/productpage created +out=$(snip_config_all_v1 2>&1) +expected="virtualservice.networking.istio.io/productpage created virtualservice.networking.istio.io/reviews created virtualservice.networking.istio.io/ratings created -virtualservice.networking.istio.io/details created -# $endsnippet +virtualservice.networking.istio.io/details created" +_verify_same "$out" "$expected" "snip_config_all_v1" # Step 2: verify no rating stars visible, (reviews-v3 traffic=0%) + verify_traffic_shift 0 # Step 3: switch 50% traffic to v3 -# $snippet -snip_config_50_v3 -# $verify -virtualservice.networking.istio.io/reviews configured -# $endsnippet +out=$(snip_config_50_v3 2>&1) +_verify_same "$out" "virtualservice.networking.istio.io/reviews configured" "snip_config_50_v3" istioctl experimental wait --for=distribution VirtualService reviews.default # Step 4: Confirm the rule was replaced -# $snippet -snip_verify_config_50_v3 -# $verify verifier="contains" -subset: v3 -# $endsnippet +out=$(snip_verify_config_50_v3 2>&1) +_verify_contains "$out" "subset: v3" "snip_verify_config_50_v3" # Step 5: verify rating stars visible 50% of the time + verify_traffic_shift 50 # Step 6: route 100% traffic to v3 -# $snippet snip_config_100_v3 -# $endsnippet verify_traffic_shift 100 diff --git a/tests/util/verify.sh b/tests/util/verify.sh new file mode 100644 index 0000000000..880b538073 --- /dev/null +++ b/tests/util/verify.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_err_exit() { + echo "VERIFY FAILED $1: $2"; + exit 1 +} + +# Verify that $out is the same as $expected. +_verify_same() { + local out=$1 + local expected=$2 + local msg=$3 + if [ "$out" != "$expected" ]; then + _err_exit "$msg" "$out" + fi +} + +# Verify that $out contains the substring $expected. +_verify_contains() { + local out=$1 + local expected=$2 + local msg=$3 + + if [[ "$out" != *"$expected"* ]]; then + _err_exit "$msg" "$out" + fi +} + +# Verify that $out is "like" $expected. Like implies: +# 1. Same number of lines +# 2. Same number of whitespace-seperated tokens per line +# 3. Tokens can only differ in the following ways: +# - different elapsed time values +# - different ip values +# - prefix match ending with a dash character +_verify_like() { + local out=$1 + local expected=$2 + local msg=$3 + + if [[ "$out" != "$expected" ]]; then + local olines=() + while read -r line; do + olines+=("$line") + done <<< "$out" + + local elines=() + while read -r line; do + elines+=("$line") + done <<< "$expected" + + if [[ ${#olines[@]} -ne ${#elines[@]} ]]; then + _err_exit "$msg" "$out" + fi + + for i in "${!olines[@]}"; do + local oline=${olines[i]} + local eline=${elines[i]} + + if [[ "$oline" == "$eline" ]]; then + continue + fi + + read -r -a otokens <<< "$oline" + read -r -a etokens <<< "$eline" + + if [[ ${#otokens[@]} -ne ${#etokens[@]} ]]; then + _err_exit "$msg" "$out" + fi + + for j in "${!otokens[@]}"; do + local otok=${otokens[j]} + local etok=${etokens[j]} + + if [[ "$otok" == "$etok" ]]; then + continue + fi + + if [[ "$otok" =~ ^[0-9]+[smhd]$ && "$etok" =~ ^[0-9]+[smhd]$ ]]; then + continue + fi + + if [[ "$otok" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ && "$etok" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + continue + fi + + local comm="" + for ((k=0; k < ${#otok}; k++)) do + if [ "${otok:$k:1}" = "${etok:$k:1}" ]; then + comm=${comm}${otok:$k:1} + else + if [[ "$comm" =~ ^([a-zA-Z0-9_]+-)+ ]]; then + break + fi + _err_exit "$msg" "$out" + fi + done + done + done + fi +}