mirror of https://github.com/istio/istio.io.git
Cleanup circuit breaking test (#7335)
* Cleanup circuit breaking test * remove old files * updated expected
This commit is contained in:
Frank Budinsky
GitHub
parent
0d633a555e
commit
90103ae557
|
|
@ -61,11 +61,9 @@ when calling the `httpbin` service:
|
|||
|
||||
{{< text bash yaml >}}
|
||||
$ kubectl get destinationrule httpbin -o yaml
|
||||
apiVersion: networking.istio.io/v1alpha3
|
||||
apiVersion: networking.istio.io/v1beta1
|
||||
kind: DestinationRule
|
||||
metadata:
|
||||
name: httpbin
|
||||
...
|
||||
...
|
||||
spec:
|
||||
host: httpbin
|
||||
trafficPolicy:
|
||||
|
|
@ -76,9 +74,9 @@ when calling the `httpbin` service:
|
|||
tcp:
|
||||
maxConnections: 1
|
||||
outlierDetection:
|
||||
baseEjectionTime: 180.000s
|
||||
baseEjectionTime: 3m
|
||||
consecutiveErrors: 1
|
||||
interval: 1.000s
|
||||
interval: 1s
|
||||
maxEjectionPercent: 100
|
||||
{{< /text >}}
|
||||
|
||||
|
|
|
|||
|
|
@ -48,11 +48,9 @@ kubectl get destinationrule httpbin -o yaml
|
|||
}
|
||||
|
||||
! read -r -d '' snip_configuring_the_circuit_breaker_2_out <<\ENDSNIP
|
||||
apiVersion: networking.istio.io/v1alpha3
|
||||
apiVersion: networking.istio.io/v1beta1
|
||||
kind: DestinationRule
|
||||
metadata:
|
||||
name: httpbin
|
||||
...
|
||||
...
|
||||
spec:
|
||||
host: httpbin
|
||||
trafficPolicy:
|
||||
|
|
@ -63,9 +61,9 @@ spec:
|
|||
tcp:
|
||||
maxConnections: 1
|
||||
outlierDetection:
|
||||
baseEjectionTime: 180.000s
|
||||
baseEjectionTime: 3m
|
||||
consecutiveErrors: 1
|
||||
interval: 1.000s
|
||||
interval: 1s
|
||||
maxEjectionPercent: 100
|
||||
ENDSNIP
|
||||
|
||||
|
|
|
|||
|
|
@ -16,19 +16,27 @@ package trafficmanagement
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"istio.io/istio.io/pkg/test/istioio"
|
||||
"istio.io/istio/pkg/test/framework"
|
||||
|
||||
"istio.io/istio.io/pkg/test/istioio"
|
||||
)
|
||||
|
||||
func TestCircuitBreaker(t *testing.T) {
|
||||
func TestCircuitBreaking(t *testing.T) {
|
||||
framework.
|
||||
NewTest(t).
|
||||
Run(istioio.NewBuilder("tasks__traffic_management__circuit_breaking").
|
||||
Add(
|
||||
istioio.Script{Input: istioio.Path("scripts/circuitbreaker_test_setup.txt")},
|
||||
istioio.MultiPodWait("istio-io-circuitbreaker"),
|
||||
istioio.Script{Input: istioio.Path("scripts/trip_circuitbreaker.txt")},
|
||||
).
|
||||
Defer(istioio.Script{Input: istioio.Path("scripts/circuitbreaker_test_cleanup.txt")}).
|
||||
Add(istioio.Script{
|
||||
Input: istioio.Path("scripts/circuit_breaking.sh"),
|
||||
}).
|
||||
Defer(istioio.Script{
|
||||
Input: istioio.Inline{
|
||||
FileName: "cleanup.sh",
|
||||
Value: `
|
||||
set +e # ignore cleanup errors
|
||||
source ${REPO_ROOT}/content/en/docs/tasks/traffic-management/circuit-breaking/snips.sh
|
||||
snip_cleaning_up_1
|
||||
snip_cleaning_up_2`,
|
||||
},
|
||||
}).
|
||||
Build())
|
||||
}
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
#!/usr/bin/env bash
|
||||
# shellcheck disable=SC1090,SC2154
|
||||
|
||||
# Copyright 2020 Istio Authors
|
||||
#
|
||||
|
|
@ -18,9 +19,27 @@ set -e
|
|||
set -u
|
||||
set -o pipefail
|
||||
|
||||
FORTIO_POD=$(kubectl get pods -n istio-io-circuitbreaker | grep fortio | awk '{ print $1 }')
|
||||
out=$(kubectl -n istio-io-circuitbreaker exec $FORTIO_POD -c fortio -- /usr/bin/fortio load -curl http://httpbin:8000/get 2>&1)
|
||||
_verify_contains "$out" "200 OK" "test_fortio_httpbin_interaction"
|
||||
source "${REPO_ROOT}/content/en/docs/tasks/traffic-management/circuit-breaking/snips.sh"
|
||||
source "${REPO_ROOT}/tests/util/samples.sh"
|
||||
|
||||
kubectl label namespace default istio-injection=enabled --overwrite
|
||||
|
||||
# Launch the httpbin sample
|
||||
startup_httpbin_sample
|
||||
|
||||
# Create destination rule
|
||||
snip_configuring_the_circuit_breaker_1
|
||||
|
||||
# Confirm destination rule set
|
||||
_run_and_verify_elided snip_configuring_the_circuit_breaker_2 "$snip_configuring_the_circuit_breaker_2_out"
|
||||
|
||||
# Deploy fortio client
|
||||
snip_adding_a_client_1
|
||||
|
||||
sample_wait_for_deployment default fortio-deploy
|
||||
|
||||
# Make one call to httpbin
|
||||
_run_and_verify_contains snip_adding_a_client_3 "HTTP/1.1 200 OK"
|
||||
|
||||
# FIXME / TODO: These tests previously relied on checking that the
|
||||
# percentage of 200 and 503 responses fell within a given range. That
|
||||
|
|
@ -32,22 +51,22 @@ _verify_contains "$out" "200 OK" "test_fortio_httpbin_interaction"
|
|||
# Temporary fix: https://github.com/istio/istio.io/pull/7043
|
||||
# Issue: https://github.com/istio/istio.io/issues/7074
|
||||
|
||||
out=$(kubectl -n istio-io-circuitbreaker exec $FORTIO_POD -c fortio -- /usr/bin/fortio \
|
||||
load -c 2 -qps 0 -n 20 -loglevel warning http://httpbin:8000/get 2>&1)
|
||||
_verify_contains "$out" "Code 200 :" "almost_trip_circuit_breaker"
|
||||
_verify_contains "$out" "Code 503 :" "almost_trip_circuit_breaker"
|
||||
# Make requests with 2 connections
|
||||
out=$(snip_tripping_the_circuit_breaker_1 2>&1)
|
||||
_verify_contains "$out" "Code 200 :" "snip_tripping_the_circuit_breaker_1"
|
||||
_verify_contains "$out" "Code 503 :" "snip_tripping_the_circuit_breaker_1"
|
||||
|
||||
out=$(kubectl -n istio-io-circuitbreaker exec $FORTIO_POD -c fortio -- /usr/bin/fortio \
|
||||
load -c 3 -qps 0 -n 30 -loglevel warning http://httpbin:8000/get 2>&1)
|
||||
_verify_contains "$out" "Code 200 :" "trip_circuit_breaker"
|
||||
_verify_contains "$out" "Code 503 :" "trip_circuit_breaker"
|
||||
# Make requests with 3 connections
|
||||
out=$(snip_tripping_the_circuit_breaker_3 2>&1)
|
||||
_verify_contains "$out" "Code 200 :" "snip_tripping_the_circuit_breaker_3"
|
||||
_verify_contains "$out" "Code 503 :" "snip_tripping_the_circuit_breaker_3"
|
||||
|
||||
out=$(kubectl -n istio-io-circuitbreaker exec $FORTIO_POD -c istio-proxy -- \
|
||||
pilot-agent request GET stats | grep httpbin | grep pending 2>&1)
|
||||
# Query the istio-proxy stats
|
||||
out=$(snip_tripping_the_circuit_breaker_5 2>&1)
|
||||
expected="cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.circuit_breakers.default.rq_pending_open: ...
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.circuit_breakers.high.rq_pending_open: ...
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_active: ...
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_failure_eject: ...
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_overflow: ...
|
||||
cluster.outbound|8000||httpbin.istio-io-circuitbreaker.svc.cluster.local.upstream_rq_pending_total: ..."
|
||||
_verify_like "$out" "$expected" "print_statistics_after_tripping"
|
||||
_verify_like "$out" "$expected" "snip_tripping_the_circuit_breaker_5"
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2020 Istio Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
kubectl delete destinationrule httpbin -n istio-io-circuitbreaker
|
||||
kubectl delete deploy httpbin fortio-deploy -n istio-io-circuitbreaker
|
||||
kubectl delete svc httpbin -n istio-io-circuitbreaker
|
||||
kubectl delete ns istio-io-circuitbreaker
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2020 Istio Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
cat<<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: istio-io-circuitbreaker
|
||||
labels:
|
||||
istio-injection: enabled
|
||||
EOF
|
||||
|
||||
kubectl apply -f samples/httpbin/httpbin.yaml -n istio-io-circuitbreaker
|
||||
|
||||
cat<<EOF | kubectl -n istio-io-circuitbreaker apply -f -
|
||||
apiVersion: networking.istio.io/v1alpha3
|
||||
kind: DestinationRule
|
||||
metadata:
|
||||
name: httpbin
|
||||
spec:
|
||||
host: httpbin
|
||||
trafficPolicy:
|
||||
connectionPool:
|
||||
tcp:
|
||||
maxConnections: 1
|
||||
http:
|
||||
http1MaxPendingRequests: 1
|
||||
maxRequestsPerConnection: 1
|
||||
outlierDetection:
|
||||
consecutiveErrors: 1
|
||||
interval: 1s
|
||||
baseEjectionTime: 3m
|
||||
maxEjectionPercent: 100
|
||||
EOF
|
||||
|
||||
out=$(kubectl get destinationrule httpbin -n istio-io-circuitbreaker -o yaml 2>&1)
|
||||
expected="spec:
|
||||
host: httpbin
|
||||
trafficPolicy:
|
||||
connectionPool:
|
||||
http:
|
||||
http1MaxPendingRequests: 1
|
||||
maxRequestsPerConnection: 1
|
||||
tcp:
|
||||
maxConnections: 1
|
||||
outlierDetection:
|
||||
baseEjectionTime: 3m
|
||||
consecutiveErrors: 1
|
||||
interval: 1s
|
||||
maxEjectionPercent: 100"
|
||||
_verify_contains "$out" "$expected" "setup_destinationrule_httpbin"
|
||||
|
||||
kubectl apply -f samples/httpbin/sample-client/fortio-deploy.yaml -n istio-io-circuitbreaker
|
||||
|
|
@ -195,7 +195,7 @@ __verify_with_retry() {
|
|||
local cmd=$2
|
||||
local expected=$3
|
||||
local max_attempts=$4
|
||||
local attempt=0
|
||||
local attempt=1
|
||||
|
||||
while true; do
|
||||
out=$($cmd 2>&1)
|
||||
|
|
@ -217,54 +217,59 @@ __verify_with_retry() {
|
|||
|
||||
# Runs $func and compares the output with $expected. If they are not the same,
|
||||
# will retry $max_attempts times with an exponential backoff. If $max_attempts
|
||||
# is not set, a retry will not be attempted.
|
||||
# is not set, it will retry 5 times by default.
|
||||
_run_and_verify_same() {
|
||||
local func=$1
|
||||
local expected=$2
|
||||
local max_attempts=${3:-0}
|
||||
local max_attempts=${3:-5}
|
||||
__verify_with_retry __cmp_same "$func" "$expected" "$max_attempts"
|
||||
}
|
||||
|
||||
# Runs $func and compares the output with $expected. If the output does not
|
||||
# contain the substring $expected, will retry $max_attempts times with an
|
||||
# exponential backoff. If $max_attempts is not set, a retry will not be
|
||||
# attempted.
|
||||
# exponential backoff. If $max_attempts is not set, it will retry 5 times by
|
||||
# default.
|
||||
_run_and_verify_contains() {
|
||||
local func=$1
|
||||
local expected=$2
|
||||
local max_attempts=${3:-0}
|
||||
local max_attempts=${3:-5}
|
||||
__verify_with_retry __cmp_contains "$func" "$expected" "$max_attempts"
|
||||
}
|
||||
|
||||
# Runs $func and compares the output with $expected. If the output contains the
|
||||
# substring $expected, will retry $max_attempts times with an exponential
|
||||
# backoff. If $max_attempts is not set, a retry will not be attempted.
|
||||
# backoff. If $max_attempts is not set, it will retry 5 times by default.
|
||||
#
|
||||
# This function is not useful since __cmp_not_contains will return true
|
||||
# even if the function fails. The _verify_not_contains function, itself,
|
||||
# is also often not very useful for the same reason.
|
||||
# TODO Replace it with some kind of _verify_worked_and_not_contains function.
|
||||
_run_and_verify_not_contains() {
|
||||
local func=$1
|
||||
local expected=$2
|
||||
local max_attempts=${3:-0}
|
||||
local max_attempts=${3:-5}
|
||||
__verify_with_retry __cmp_not_contains "$func" "$expected" "$max_attempts"
|
||||
}
|
||||
|
||||
# Runs $func and compares the output with $expected. If the output does not
|
||||
# contain the lines in $expected where "..." on a line matches one or more lines
|
||||
# containing any text, will retry $max_attempts times with an exponential
|
||||
# backoff. If $max_attempts is not set, a retry will not be attempted.
|
||||
# backoff. If $max_attempts is not set, it will retry 5 times by default.
|
||||
_run_and_verify_elided() {
|
||||
local func=$1
|
||||
local expected=$2
|
||||
local max_attempts=${3:-0}
|
||||
local max_attempts=${3:-5}
|
||||
__verify_with_retry __cmp_elided "$func" "$expected" "$max_attempts"
|
||||
}
|
||||
|
||||
# Runs $func and compares the output with $expected. If the first line of
|
||||
# output does not match the first line in $expected, will retry $max_attempts
|
||||
# times with an exponential backoff. If $max_attempts is not set, a retry will
|
||||
# not be attempted.
|
||||
# times with an exponential backoff. If $max_attempts is not set, it will
|
||||
# retry 5 times by default.
|
||||
_run_and_verify_first_line() {
|
||||
local func=$1
|
||||
local expected=$2
|
||||
local max_attempts=${3:-0}
|
||||
local max_attempts=${3:-5}
|
||||
__verify_with_retry __cmp_first_line "$func" "$expected" "$max_attempts"
|
||||
}
|
||||
|
||||
|
|
@ -278,11 +283,11 @@ _run_and_verify_first_line() {
|
|||
# - different ip values
|
||||
# - prefix match ending with a dash character
|
||||
# - expected ... is a wildcard token, matches anything
|
||||
# If $max_attempts is not set, a retry will not be attempted.
|
||||
# If $max_attempts is not set, it will retry 5 times by default.
|
||||
_run_and_verify_like() {
|
||||
local func=$1
|
||||
local expected=$2
|
||||
local max_attempts=${3:-0}
|
||||
local max_attempts=${3:-5}
|
||||
__verify_with_retry __cmp_like "$func" "$expected" "$max_attempts"
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue