mirror of https://github.com/istio/istio.io.git
* (#6586) move istio.io test for circuit breaking to istio.io repository * (#6586) add verification for output metrics * (#6586) widened bounds for both before and after tripping and use ruby for evaluation * (#6586) fix istioio package import
This commit is contained in:
Suchith J N
GitHub
parent
782a7dba4c
commit
798328b911
|
|
@ -0,0 +1,48 @@
|
|||
// Copyright 2020 Istio Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package trafficmanagement
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"istio.io/istio.io/pkg/test/istioio"
|
||||
"istio.io/istio/pkg/test/framework"
|
||||
)
|
||||
|
||||
func TestCircuitBreaker(t *testing.T) {
|
||||
framework.
|
||||
NewTest(t).
|
||||
Run(istioio.NewBuilder("tasks__traffic_management__circuit_breaking").
|
||||
Add(
|
||||
istioio.Script{Input: istioio.Path("scripts/circuitbreaker_test_setup.txt")},
|
||||
istioio.MultiPodWait("istio-io-circuitbreaker"),
|
||||
istioio.Script{
|
||||
Input: istioio.Evaluate(istioio.Path("scripts/trip_circuitbreaker.txt"), map[string]interface{}{
|
||||
"isSnippet": false,
|
||||
"inputTerminalFlag": "",
|
||||
"beforeCircuitBreakVerify": " 2>&1 | verify_circuit_breaking 60 100 0 40",
|
||||
"afterCircuitBreakVerify": " 2>&1 | verify_circuit_breaking 20 80 20 80",
|
||||
"outputRedirectionCmd": "2>&1",
|
||||
}),
|
||||
SnippetInput: istioio.Evaluate(istioio.Path("scripts/trip_circuitbreaker.txt"), map[string]interface{}{
|
||||
"isSnippet": true,
|
||||
"inputTerminalFlag": "-it",
|
||||
"beforeCircuitBreakVerify": "",
|
||||
"afterCircuitBreakVerify": "",
|
||||
"outputRedirectionCmd": "",
|
||||
}),
|
||||
}).
|
||||
Defer(istioio.Script{Input: istioio.Path("scripts/circuitbreaker_test_cleanup.txt")}).
|
||||
Build())
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2020 Istio Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
# $snippet cleanup.sh syntax="bash"
|
||||
$ kubectl delete destinationrule httpbin -n istio-io-circuitbreaker
|
||||
$ kubectl delete deploy httpbin fortio-deploy -n istio-io-circuitbreaker
|
||||
$ kubectl delete svc httpbin -n istio-io-circuitbreaker
|
||||
$ kubectl delete ns istio-io-circuitbreaker
|
||||
# $endsnippet
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2020 Istio Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
cat<<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: istio-io-circuitbreaker
|
||||
labels:
|
||||
istio-injection: enabled
|
||||
EOF
|
||||
|
||||
# $snippet deploy_httpbin.sh syntax="bash"
|
||||
$ kubectl apply -f @samples/httpbin/httpbin.yaml@ -n istio-io-circuitbreaker
|
||||
# $endsnippet
|
||||
|
||||
# $snippet deploy_httpbin_destination_rule.sh syntax="bash"
|
||||
$ cat<<EOF | kubectl -n istio-io-circuitbreaker apply -f -
|
||||
apiVersion: networking.istio.io/v1alpha3
|
||||
kind: DestinationRule
|
||||
metadata:
|
||||
name: httpbin
|
||||
spec:
|
||||
host: httpbin
|
||||
trafficPolicy:
|
||||
connectionPool:
|
||||
tcp:
|
||||
maxConnections: 1
|
||||
http:
|
||||
http1MaxPendingRequests: 1
|
||||
maxRequestsPerConnection: 1
|
||||
outlierDetection:
|
||||
consecutiveErrors: 1
|
||||
interval: 1s
|
||||
baseEjectionTime: 3m
|
||||
maxEjectionPercent: 100
|
||||
EOF
|
||||
|
||||
$ kubectl get destinationrule httpbin -n istio-io-circuitbreaker -o yaml
|
||||
# $verify verifier="contains"
|
||||
spec:
|
||||
host: httpbin
|
||||
trafficPolicy:
|
||||
connectionPool:
|
||||
http:
|
||||
http1MaxPendingRequests: 1
|
||||
maxRequestsPerConnection: 1
|
||||
tcp:
|
||||
maxConnections: 1
|
||||
outlierDetection:
|
||||
baseEjectionTime: 3m
|
||||
consecutiveErrors: 1
|
||||
interval: 1s
|
||||
maxEjectionPercent: 100
|
||||
# $endsnippet
|
||||
|
||||
# $snippet deploy_fortio.sh syntax="bash"
|
||||
$ kubectl apply -f @samples/httpbin/sample-client/fortio-deploy.yaml@ -n istio-io-circuitbreaker
|
||||
# $endsnippet
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2020 Istio Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
# $snippet test_fortio_httpbin_interaction.sh syntax="bash" outputis="text"
|
||||
$ FORTIO_POD=$(kubectl get pods -n istio-io-circuitbreaker | grep fortio | awk '{ print $1 }')
|
||||
$ kubectl -n istio-io-circuitbreaker exec {{ .inputTerminalFlag }} $FORTIO_POD -c fortio /usr/bin/fortio -- load -curl http://httpbin:8000/get
|
||||
# $verify verifier="contains"
|
||||
200 OK
|
||||
# $endsnippet
|
||||
|
||||
function ruby_eval() {
|
||||
ruby -e "p $1"
|
||||
}
|
||||
|
||||
# check_percentage_bounds take 4 arguments as input. It must be invoked as follows
|
||||
#
|
||||
# $ check_percentage_bounds <percentage-lower-bound-200> <percentage-upper-bound-200> \
|
||||
# <percentage-upper-bound-503> <percentage-upper-bound-503>
|
||||
#
|
||||
# Percentage of requests with status 200 must be between "percentage-lower-bound-200" and
|
||||
# "percentage-upper-bound-200" inclusive. Same for 503. If bounds are breached then the
|
||||
# function exits with status 1
|
||||
function check_percentage_bounds() {
|
||||
local lower_bounds=( ["200"]=${1:-"100"} ["503"]=${3:-"0"} )
|
||||
local upper_bounds=( ["200"]=${2:-"100"} ["503"]=${4:-"0"} )
|
||||
|
||||
readarray -t code_lines < <(grep -E "^Code ([1-5][0-9]{2,}) : [0-9]+ \(([0-9\.]+) %\)$")
|
||||
|
||||
for code_line in "${code_lines[@]}"; do
|
||||
local status_code=$(echo $code_line | cut -d' ' -f2)
|
||||
local percentage_req=$(echo $code_line | cut -d' ' -f5 | sed "s/[\(]//g")
|
||||
|
||||
local status_lower_bound=${lower_bounds[$status_code]}
|
||||
local status_upper_bound=${upper_bounds[$status_code]}
|
||||
|
||||
local is_lower_bound_breached=$(ruby_eval "$percentage_req < $status_lower_bound")
|
||||
local is_upper_bound_breached=$(ruby_eval "$percentage_req > $status_upper_bound")
|
||||
|
||||
if [[ $is_lower_bound_breached == "true" || $is_upper_bound_breached == "true" ]]; then
|
||||
echo "either lower bound or upper bound is breached"
|
||||
echo "status=$status_code actual=$percentage_req, low=$status_lower_bound high=$status_upper_bound"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function verify_circuit_breaking() {
|
||||
local fortio_output=`cat` # preserve output
|
||||
|
||||
echo "$fortio_output" | check_percentage_bounds $1 $2 $3 $4
|
||||
if (($? != 0)); then
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
echo "$fortio_output"
|
||||
}
|
||||
|
||||
# $snippet almost_trip_circuit_breaker.sh syntax="bash" outputis="text"
|
||||
$ kubectl -n istio-io-circuitbreaker exec {{ .inputTerminalFlag }} $FORTIO_POD -c fortio /usr/bin/fortio -- \
|
||||
load -c 2 -qps 0 -n 20 -loglevel warning http://httpbin:8000/get {{ .beforeCircuitBreakVerify }}
|
||||
# $endsnippet
|
||||
|
||||
|
||||
# $snippet trip_circuit_breaker.sh syntax="bash" outputis="text"
|
||||
$ kubectl -n istio-io-circuitbreaker exec {{ .inputTerminalFlag }} $FORTIO_POD -c fortio /usr/bin/fortio -- \
|
||||
load -c 3 -qps 0 -n 30 -loglevel warning http://httpbin:8000/get {{ .afterCircuitBreakVerify }}
|
||||
# $endsnippet
|
||||
|
||||
# $snippet print_statistics_after_tripping.sh syntax="bash" outputis="text"
|
||||
$ kubectl -n istio-io-circuitbreaker exec {{ .inputTerminalFlag }} $FORTIO_POD -c istio-proxy -- \
|
||||
pilot-agent request GET stats | grep httpbin | grep pending {{ .outputRedirectionCmd }}
|
||||
# $verify
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.circuit_breakers.default.rq_pending_open: ?
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.circuit_breakers.high.rq_pending_open: ?
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_active: ?
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_failure_eject: ?
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_overflow: ?
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_total: ?
|
||||
# $endsnippet
|
||||
Loading…
Reference in New Issue