containers
/
automation-tests
mirror of https://github.com/containers/automation-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

minikube: instrument tests, to allow debugging failures 

New run_minikube() helper, modeled after run_podman(). Echoes
each command being run and its output. On failure, runs minikube logs.

Addresses (does not close) #21931 which is hitting us hard in CI.
Probably quay flakes, but it's impossible to tell without logs.

Also: bug fix: one "run podman" fixed to run_podman

Signed-off-by: Ed Santiago <santiago@redhat.com>
This commit is contained in:
Ed Santiago 2024-03-19 10:02:43 -06:00
parent 8a643c243e
commit aa5c4879b4
2 changed files with 91 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
test/minikube/001-kube.bats
View File

@ -9,11 +9,9 @@ load helpers.bash
# BEGIN tests # BEGIN tests
@test "minikube - check cluster is up" { @test "minikube - check cluster is up" {
run minikube kubectl get nodes run_minikube kubectl get nodes
assert "$status" -eq 0 "get status of nodes"
assert "$output" =~ "Ready" assert "$output" =~ "Ready"
run minikube kubectl get pods run_minikube kubectl get pods
assert "$status" -eq 0 "get pods in the default namespace"
assert "$output" == "No resources found in default namespace." assert "$output" == "No resources found in default namespace."
} }
@ -25,14 +23,11 @@ load helpers.bash
# deploy to the minikube cluster # deploy to the minikube cluster
project="ctr-ns" project="ctr-ns"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project" run_minikube kubectl -- apply -f $fname
run minikube kubectl -- apply -f $fname
assert "$status" -eq 0 "deploy $fname to the cluster"
assert "$output" == "pod/$cname-pod created" assert "$output" == "pod/$cname-pod created"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - deploy generated pod yaml to minikube" { @test "minikube - deploy generated pod yaml to minikube" {
@ -48,14 +43,11 @@ load helpers.bash
# deploy to the minikube cluster # deploy to the minikube cluster
project="pod-ns" project="pod-ns"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project" run_minikube kubectl -- apply -f $fname
run minikube kubectl -- apply -f $fname
assert "$status" -eq 0 "deploy $fname to the cluster"
assert "$output" == "pod/$pname created" assert "$output" == "pod/$pname created"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - apply podman ctr to cluster" { @test "minikube - apply podman ctr to cluster" {
@ -64,35 +56,29 @@ load helpers.bash
# deploy to minikube cluster with kube apply # deploy to minikube cluster with kube apply
project="ctr-apply" project="ctr-apply"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project"
run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $cname run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $cname
assert "$output" =~ "Successfully deployed workloads to cluster!" assert "$output" =~ "Successfully deployed workloads to cluster!"
run minikube kubectl -- get pods --namespace $project run_minikube kubectl -- get pods --namespace $project
assert "$status" -eq 0 "kube apply $cname to the cluster"
assert "$output" =~ "$cname-pod" assert "$output" =~ "$cname-pod"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - apply podman pod to cluster" { @test "minikube - apply podman pod to cluster" {
pname="test-pod-apply" pname="test-pod-apply"
run_podman pod create --name $pname run_podman pod create --name $pname
run podman container create --pod $pname $IMAGE top run_podman container create --pod $pname $IMAGE top
# deploy to minikube cluster with kube apply # deploy to minikube cluster with kube apply
project="pod-apply" project="pod-apply"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project"
run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $pname run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $pname
assert "$output" =~ "Successfully deployed workloads to cluster!" assert "$output" =~ "Successfully deployed workloads to cluster!"
run minikube kubectl -- get pods --namespace $project run_minikube kubectl -- get pods --namespace $project
assert "$status" -eq 0 "kube apply $pname to the cluster"
assert "$output" =~ "$pname" assert "$output" =~ "$pname"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - deploy generated kube yaml with podman kube apply to cluster" { @test "minikube - deploy generated kube yaml with podman kube apply to cluster" {
@ -108,16 +94,13 @@ load helpers.bash
# deploy to minikube cluster with kube apply # deploy to minikube cluster with kube apply
project="yaml-apply" project="yaml-apply"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project"
run_podman kube apply --kubeconfig $KUBECONFIG --ns $project -f $fname run_podman kube apply --kubeconfig $KUBECONFIG --ns $project -f $fname
assert "$output" =~ "Successfully deployed workloads to cluster!" assert "$output" =~ "Successfully deployed workloads to cluster!"
run minikube kubectl -- get pods --namespace $project run_minikube kubectl -- get pods --namespace $project
assert "$status" -eq 0 "kube apply $pname to the cluster"
assert "$output" =~ "$pname" assert "$output" =~ "$pname"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - apply podman ctr with volume to cluster" { @test "minikube - apply podman ctr with volume to cluster" {
@ -127,19 +110,15 @@ load helpers.bash
# deploy to minikube cluster with kube apply # deploy to minikube cluster with kube apply
project="ctr-vol-apply" project="ctr-vol-apply"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project"
run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $cname $vname run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $cname $vname
assert "$output" =~ "Successfully deployed workloads to cluster!" assert "$output" =~ "Successfully deployed workloads to cluster!"
run minikube kubectl -- get pods --namespace $project run_minikube kubectl -- get pods --namespace $project
assert "$status" -eq 0 "kube apply $cname to the cluster"
assert "$output" =~ "$cname-pod" assert "$output" =~ "$cname-pod"
run minikube kubectl -- get pvc --namespace $project run_minikube kubectl -- get pvc --namespace $project
assert "$status" -eq 0 "kube apply $vname to the cluster"
assert "$output" =~ "$vname" assert "$output" =~ "$vname"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - apply podman ctr with service to cluster" { @test "minikube - apply podman ctr with service to cluster" {
@ -148,19 +127,15 @@ load helpers.bash
# deploy to minikube cluster with kube apply # deploy to minikube cluster with kube apply
project="ctr-svc-apply" project="ctr-svc-apply"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project"
run_podman kube apply --kubeconfig $KUBECONFIG -s --ns $project $cname run_podman kube apply --kubeconfig $KUBECONFIG -s --ns $project $cname
assert "$output" =~ "Successfully deployed workloads to cluster!" assert "$output" =~ "Successfully deployed workloads to cluster!"
run minikube kubectl -- get pods --namespace $project run_minikube kubectl -- get pods --namespace $project
assert "$status" -eq 0 "kube apply $cname to the cluster"
assert "$output" =~ "$cname-pod" assert "$output" =~ "$cname-pod"
run minikube kubectl -- get svc --namespace $project run_minikube kubectl -- get svc --namespace $project
assert "$status" -eq 0 "kube apply service to the cluster"
assert "$output" =~ "$cname-pod" assert "$output" =~ "$cname-pod"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - deploy generated container yaml to minikube --type=deployment" { @test "minikube - deploy generated container yaml to minikube --type=deployment" {
@ -171,14 +146,11 @@ load helpers.bash
# deploy to the minikube cluster # deploy to the minikube cluster
project="dep-ctr-ns" project="dep-ctr-ns"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project" run_minikube kubectl -- apply -f $fname
run minikube kubectl -- apply -f $fname
assert "$status" -eq 0 "deploy $fname to the cluster"
assert "$output" == "deployment.apps/$cname-pod-deployment created" assert "$output" == "deployment.apps/$cname-pod-deployment created"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - deploy generated pod yaml to minikube --type=deployment" { @test "minikube - deploy generated pod yaml to minikube --type=deployment" {
@ -194,14 +166,11 @@ load helpers.bash
# deploy to the minikube cluster # deploy to the minikube cluster
project="dep-pod-ns" project="dep-pod-ns"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project" run_minikube kubectl -- apply -f $fname
run minikube kubectl -- apply -f $fname
assert "$status" -eq 0 "deploy $fname to the cluster"
assert "$output" == "deployment.apps/$pname-deployment created" assert "$output" == "deployment.apps/$pname-deployment created"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - deploy generated container yaml to minikube --type=daemonset" { @test "minikube - deploy generated container yaml to minikube --type=daemonset" {
@ -212,14 +181,11 @@ load helpers.bash
# deploy to the minikube cluster # deploy to the minikube cluster
project="dep-ctr-ns" project="dep-ctr-ns"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project" run_minikube kubectl -- apply -f $fname
run minikube kubectl -- apply -f $fname
assert "$status" -eq 0 "deploy $fname to the cluster"
assert "$output" == "daemonset.apps/$cname-pod-daemonset created" assert "$output" == "daemonset.apps/$cname-pod-daemonset created"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }
@test "minikube - deploy generated pod yaml to minikube --type=daemonset" { @test "minikube - deploy generated pod yaml to minikube --type=daemonset" {
@ -235,12 +201,9 @@ load helpers.bash
# deploy to the minikube cluster # deploy to the minikube cluster
project="dep-pod-ns" project="dep-pod-ns"
run minikube kubectl create namespace $project run_minikube kubectl create namespace $project
assert "$status" -eq 0 "create new namespace $project" run_minikube kubectl -- apply -f $fname
run minikube kubectl -- apply -f $fname
assert "$status" -eq 0 "deploy $fname to the cluster"
assert "$output" == "daemonset.apps/$pname-daemonset created" assert "$output" == "daemonset.apps/$pname-daemonset created"
wait_for_pods_to_start wait_for_pods_to_start
run minikube kubectl delete namespace $project run_minikube kubectl delete namespace $project
assert $status -eq 0 "delete namespace $project"
} }

58
test/minikube/helpers.bash
View File

@ -4,10 +4,55 @@ load ../system/helpers.bash
KUBECONFIG="$HOME/.kube/config" KUBECONFIG="$HOME/.kube/config"
##################
# run_minikube # Local helper, with instrumentation for debugging failures
##################
function run_minikube() {
# Number as first argument = expected exit code; default 0
local expected_rc=0
case "$1" in
[0-9]) expected_rc=$1; shift;;
[1-9][0-9]) expected_rc=$1; shift;;
[12][0-9][0-9]) expected_rc=$1; shift;;
'?') expected_rc= ; shift;; # ignore exit code
esac
# stdout is only emitted upon error; this printf is to help in debugging
printf "\n%s %s %s %s\n" "$(timestamp)" "\$" "minikube" "$*"
run minikube "$@"
# without "quotes", multiple lines are glommed together into one
if [[ -n "$output" ]]; then
echo "$(timestamp) $output"
fi
if [[ "$status" -ne 0 ]]; then
echo -n "$(timestamp) [ rc=$status ";
if [[ -n "$expected_rc" ]]; then
if [[ "$status" -eq "$expected_rc" ]]; then
echo -n "(expected) ";
else
echo -n "(** EXPECTED $expected_rc **) ";
fi
fi
echo "]"
fi
if [[ -n "$expected_rc" ]]; then
if [[ "$status" -ne "$expected_rc" ]]; then
# Further debugging
echo "\$ minikube logs"
run minikube logs
echo "$output"
die "exit code is $status; expected $expected_rc"
fi
fi
}
function setup(){ function setup(){
# only set up the minikube cluster before the first test # only set up the minikube cluster before the first test
if [[ "$BATS_TEST_NUMBER" -eq 1 ]]; then if [[ "$BATS_TEST_NUMBER" -eq 1 ]]; then
minikube start run_minikube start
wait_for_default_sa wait_for_default_sa
fi fi
basic_setup basic_setup
@ -17,8 +62,12 @@ function teardown(){
# only delete the minikube cluster if we are done with the last test # only delete the minikube cluster if we are done with the last test
# the $DEBUG_MINIKUBE env can be set to preserve the cluster to debug if needed # the $DEBUG_MINIKUBE env can be set to preserve the cluster to debug if needed
if [[ "$BATS_TEST_NUMBER" -eq ${#BATS_TEST_NAMES[@]} ]] && [[ "$DEBUG_MINIKUBE" == "" ]]; then if [[ "$BATS_TEST_NUMBER" -eq ${#BATS_TEST_NAMES[@]} ]] && [[ "$DEBUG_MINIKUBE" == "" ]]; then
minikube delete run_minikube delete
fi fi
# Prevents nasty red warnings in log
run_podman rmi --ignore $(pause_image)
basic_teardown basic_teardown
} }
@ -29,8 +78,7 @@ function wait_for_default_sa(){
# if the default service account hasn't been created yet, there is something else wrong # if the default service account hasn't been created yet, there is something else wrong
while [[ $count -lt 30 ]] && [[ $sa_ready == false ]] while [[ $count -lt 30 ]] && [[ $sa_ready == false ]]
do do
run minikube kubectl get sa run_minikube kubectl get sa
assert "$status" -eq 0
if [[ "$output" != "No resources found in default namespace." ]]; then if [[ "$output" != "No resources found in default namespace." ]]; then
sa_ready=true sa_ready=true
fi fi
@ -49,7 +97,7 @@ function wait_for_pods_to_start(){
# if the pod hasn't started running after 30 seconds, there is something else wrong # if the pod hasn't started running after 30 seconds, there is something else wrong
while [[ $count -lt 30 ]] && [[ $running == false ]] while [[ $count -lt 30 ]] && [[ $running == false ]]
do do
run minikube kubectl get pods run_minikube kubectl get pods
assert "$status" -eq 0 assert "$status" -eq 0
if [[ "$output" =~ "Running" ]]; then if [[ "$output" =~ "Running" ]]; then
running=true running=true