diff --git a/tests/trafficmanagement/circuitbreaker_test.go b/tests/trafficmanagement/circuitbreaker_test.go new file mode 100644 index 0000000000..9e118a12b7 --- /dev/null +++ b/tests/trafficmanagement/circuitbreaker_test.go @@ -0,0 +1,48 @@ +// Copyright 2020 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package trafficmanagement + +import ( + "testing" + + "istio.io/istio.io/pkg/test/istioio" + "istio.io/istio/pkg/test/framework" +) + +func TestCircuitBreaker(t *testing.T) { + framework. + NewTest(t). + Run(istioio.NewBuilder("tasks__traffic_management__circuit_breaking"). + Add( + istioio.Script{Input: istioio.Path("scripts/circuitbreaker_test_setup.txt")}, + istioio.MultiPodWait("istio-io-circuitbreaker"), + istioio.Script{ + Input: istioio.Evaluate(istioio.Path("scripts/trip_circuitbreaker.txt"), map[string]interface{}{ + "isSnippet": false, + "inputTerminalFlag": "", + "beforeCircuitBreakVerify": " 2>&1 | verify_circuit_breaking 60 100 0 40", + "afterCircuitBreakVerify": " 2>&1 | verify_circuit_breaking 20 80 20 80", + "outputRedirectionCmd": "2>&1", + }), + SnippetInput: istioio.Evaluate(istioio.Path("scripts/trip_circuitbreaker.txt"), map[string]interface{}{ + "isSnippet": true, + "inputTerminalFlag": "-it", + "beforeCircuitBreakVerify": "", + "afterCircuitBreakVerify": "", + "outputRedirectionCmd": "", + }), + }). + Defer(istioio.Script{Input: istioio.Path("scripts/circuitbreaker_test_cleanup.txt")}). + Build()) +} diff --git a/tests/trafficmanagement/scripts/circuitbreaker_test_cleanup.txt b/tests/trafficmanagement/scripts/circuitbreaker_test_cleanup.txt new file mode 100644 index 0000000000..0d52437614 --- /dev/null +++ b/tests/trafficmanagement/scripts/circuitbreaker_test_cleanup.txt @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Copyright 2020 Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -u +set -o pipefail + +# $snippet cleanup.sh syntax="bash" +$ kubectl delete destinationrule httpbin -n istio-io-circuitbreaker +$ kubectl delete deploy httpbin fortio-deploy -n istio-io-circuitbreaker +$ kubectl delete svc httpbin -n istio-io-circuitbreaker +$ kubectl delete ns istio-io-circuitbreaker +# $endsnippet diff --git a/tests/trafficmanagement/scripts/circuitbreaker_test_setup.txt b/tests/trafficmanagement/scripts/circuitbreaker_test_setup.txt new file mode 100644 index 0000000000..ce22165040 --- /dev/null +++ b/tests/trafficmanagement/scripts/circuitbreaker_test_setup.txt @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# Copyright 2020 Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -u +set -o pipefail + +cat< \ +# +# +# Percentage of requests with status 200 must be between "percentage-lower-bound-200" and +# "percentage-upper-bound-200" inclusive. Same for 503. If bounds are breached then the +# function exits with status 1 +function check_percentage_bounds() { + local lower_bounds=( ["200"]=${1:-"100"} ["503"]=${3:-"0"} ) + local upper_bounds=( ["200"]=${2:-"100"} ["503"]=${4:-"0"} ) + + readarray -t code_lines < <(grep -E "^Code ([1-5][0-9]{2,}) : [0-9]+ \(([0-9\.]+) %\)$") + + for code_line in "${code_lines[@]}"; do + local status_code=$(echo $code_line | cut -d' ' -f2) + local percentage_req=$(echo $code_line | cut -d' ' -f5 | sed "s/[\(]//g") + + local status_lower_bound=${lower_bounds[$status_code]} + local status_upper_bound=${upper_bounds[$status_code]} + + local is_lower_bound_breached=$(ruby_eval "$percentage_req < $status_lower_bound") + local is_upper_bound_breached=$(ruby_eval "$percentage_req > $status_upper_bound") + + if [[ $is_lower_bound_breached == "true" || $is_upper_bound_breached == "true" ]]; then + echo "either lower bound or upper bound is breached" + echo "status=$status_code actual=$percentage_req, low=$status_lower_bound high=$status_upper_bound" + exit 1 + fi + done +} + +function verify_circuit_breaking() { + local fortio_output=`cat` # preserve output + + echo "$fortio_output" | check_percentage_bounds $1 $2 $3 $4 + if (($? != 0)); then + exit 1; + fi + + echo "$fortio_output" +} + +# $snippet almost_trip_circuit_breaker.sh syntax="bash" outputis="text" +$ kubectl -n istio-io-circuitbreaker exec {{ .inputTerminalFlag }} $FORTIO_POD -c fortio /usr/bin/fortio -- \ + load -c 2 -qps 0 -n 20 -loglevel warning http://httpbin:8000/get {{ .beforeCircuitBreakVerify }} +# $endsnippet + + +# $snippet trip_circuit_breaker.sh syntax="bash" outputis="text" +$ kubectl -n istio-io-circuitbreaker exec {{ .inputTerminalFlag }} $FORTIO_POD -c fortio /usr/bin/fortio -- \ + load -c 3 -qps 0 -n 30 -loglevel warning http://httpbin:8000/get {{ .afterCircuitBreakVerify }} +# $endsnippet + +# $snippet print_statistics_after_tripping.sh syntax="bash" outputis="text" +$ kubectl -n istio-io-circuitbreaker exec {{ .inputTerminalFlag }} $FORTIO_POD -c istio-proxy -- \ + pilot-agent request GET stats | grep httpbin | grep pending {{ .outputRedirectionCmd }} +# $verify +cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.circuit_breakers.default.rq_pending_open: ? +cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.circuit_breakers.high.rq_pending_open: ? +cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_active: ? +cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_failure_eject: ? +cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_overflow: ? +cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_total: ? +# $endsnippet