linkerd
/
linkerd2
mirror of https://github.com/linkerd/linkerd2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for k3d in integration tests (#4994) 

* Add support for k3d in integration tests

KinD doesn't support setting LoadBalancer services out of the box. It can be added with some additional work, but it seems the solutions are not cross-platform.

K3d on the other hand facilitates this, so we'll be using k3d clusters for the multicluster integration test.

The current change sets the ground by generalizing some of the integration tests operations that were hard-coded to KinD.

- Added `bin/k3d` to wrap the setup and running of a pinned version of `k3d`.
- Refactored `bin/_test-helpers.sh` to account for tests to be run in either KinD or k3d.
- Renamed `bin/kind-load` to `bin/image-load` and make it more generic to load images for both KinD (default) and k3d. Also got rid of the no longer used `--images-host` option.
- Added a placeholder for the new `multicluster` test in the lists in `bin/_test-helpers.sh`. It starts by setting up two k3d clusters.

* Refactor handling of the `--multicluster` flag in integration tests (#4995)

Followup to #4994, based off of that branch (`alpeb/k3d-tests`).
This is more preliminary work previous to the more complete multicluster integration test.

- Removed the `--multicluster` flag from all the tests we had in `bin/_test-helpers.sh`, so only the new "multicluster" integration test will make use of that. Also got rid of the `TestUninstallMulticluster()` test in `install_test.go` to keep the multicluster stuff around, needed for the more complete multicluster test that will be implemented in a followup PR.
- Added "multicluster" to the list of tests in the `kind_integration.yml` workflow.
- For now, this new "multicluster" test in `run_multicluster_test()` is just running the install tests (`test/integration/install_test.go`) with the `--multicluster` flag.

Co-authored-by: Kevin Leimkuhler <kevin@kleimkuhler.com>
This commit is contained in:
Alejandro Pedraza 2020-09-25 16:33:17 -05:00 committed by GitHub
parent 2ec5245d67
commit b50ae6290d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 213 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/kind_integration.yml vendored
View File

@ -101,6 +101,7 @@ jobs:
- external-issuer
- helm-deep
- helm-upgrade
- multicluster
- uninstall
- upgrade-edge
- upgrade-stable

114
bin/_test-helpers.sh
View File

@ -6,14 +6,13 @@ set +e
##### Test setup helpers #####
export default_test_names=(deep external-issuer helm-deep helm-upgrade uninstall upgrade-edge upgrade-stable cni-calico-deep)
export default_test_names=(deep external-issuer helm-deep helm-upgrade multicluster uninstall upgrade-edge upgrade-stable cni-calico-deep)
export all_test_names=(cluster-domain "${default_test_names[*]}")
handle_input() {
export images=''
export images_host=''
export test_name=''
export skip_kind_create=''
export skip_cluster_create=''
while :
do
@ -26,7 +25,7 @@ Optionally specify a test with the --name flag: [${all_test_names[*]}]
Note: The cluster-domain test requires a cluster configuration with a custom cluster domain (see test/configs/cluster-domain.yaml)
Usage:
${0##*/} [--images] [--images-host ssh://linkerd-docker] [--name test-name] [--skip-kind-create] /path/to/linkerd
${0##*/} [--images] [--name test-name] [--skip-cluster-create] /path/to/linkerd
Examples:
# Run all tests in isolated clusters
@ -35,35 +34,22 @@ Examples:
# Run single test in isolated clusters
${0##*/} --name test-name /path/to/linkerd
# Skip KinD cluster creation and run all tests in default cluster context
${0##*/} --skip-kind-create /path/to/linkerd
# Skip KinD/k3d cluster creation and run all tests in default cluster context
${0##*/} --skip-cluster-create /path/to/linkerd
# Load images from tar files located under the 'image-archives' directory
# Note: This is primarily for CI
${0##*/} --images /path/to/linkerd
# Retrieve images from a remote docker instance and then load them into KinD
# Note: This is primarily for CI
${0##*/} --images --images-host ssh://linkerd-docker /path/to/linkerd
Available Commands:
--name: the argument to this option is the specific test to run
--skip-kind-create: skip KinD cluster creation step and run tests in an existing cluster.
--images: (Primarily for CI) use 'kind load image-archive' to load the images from local .tar files in the current directory.
--images-host: (Primarily for CI) the argument to this option is used as the remote docker instance from which images are first retrieved (using 'docker save') to be then loaded into KinD. This command requires --images."
--skip-cluster-create: skip KinD/k3d cluster creation step and run tests in an existing cluster.
--images: (Primarily for CI) load the images from local .tar files in the current directory."
exit 0
;;
--images)
images=1
;;
--images-host)
images_host=$2
if [ -z "$images_host" ]; then
echo 'Error: the argument for --images-host was not specified'
exit 1
fi
shift
;;
--name)
test_name=$2
if [ -z "$test_name" ]; then
@ -72,8 +58,8 @@ Available Commands:
fi
shift
;;
--skip-kind-create)
skip_kind_create=1
--skip-cluster-create)
skip_cluster_create=1
;;
*)
break
@ -81,11 +67,6 @@ Available Commands:
shift
done
if [ "$images_host" ] && [ -z "$images" ]; then
echo 'Error: --images-host needs to be used with --images' >&2
exit 1
fi
export linkerd_path="$1"
if [ -z "$linkerd_path" ]; then
echo "Error: path to linkerd binary is required
@ -124,7 +105,7 @@ check_linkerd_binary() {
##### Cluster helpers #####
create_cluster() {
create_kind_cluster() {
local name=$1
local config=$2
"$bindir"/kind create cluster --name "$name" --config "$test_directory"/configs/"$config".yaml --wait 300s 2>&1
@ -132,17 +113,29 @@ create_cluster() {
export context="kind-$name"
}
create_k3d_cluster() {
local name=$1
local network=$2
"$bindir"/k3d cluster create "$name" --wait --network "$network"
}
check_cluster() {
check_if_k8s_reachable
check_if_l5d_exists
}
delete_cluster() {
delete_kind_cluster() {
local name=$1
"$bindir"/kind delete cluster --name "$name" 2>&1
exit_on_err 'error deleting cluster'
}
delete_k3d_cluster() {
local name=$1
"$bindir"/k3d cluster delete "$name"
exit_on_err 'error deleting cluster'
}
cleanup_cluster() {
"$bindir"/test-cleanup "$context" > /dev/null 2>&1
exit_on_err 'error removing existing Linkerd resources'
@ -174,26 +167,62 @@ Help:
##### Test runner helpers #####
start_test() {
if [ "$1" == multicluster ]; then
start_k3d_test
else
start_kind_test "$@"
fi
}
start_kind_test() {
name=$1
config=$2
export helm_path="$bindir"/helm
test_setup
if [ -z "$skip_kind_create" ]; then
create_cluster "$name" "$config"
"$bindir"/kind-load ${images:+'--images'} ${images_host:+'--images-host' "$images_host"} "$name"
if [ -z "$skip_cluster_create" ]; then
create_kind_cluster "$name" "$config"
"$bindir"/image-load --kind ${images:+'--images'} "$name"
exit_on_err "error calling '$bindir/image-load'"
fi
check_cluster
run_"$name"_test
exit_on_err "error calling 'run_${name}_test'"
if [ -z "$skip_kind_create" ]; then
delete_cluster "$name"
if [ -z "$skip_cluster_create" ]; then
delete_kind_cluster "$name"
else
cleanup_cluster
fi
}
start_k3d_test() {
test_setup
if [ -z "$skip_cluster_create" ]; then
create_k3d_cluster source multicluster-test
"$bindir"/image-load --k3d ${images:+'--images'} source
create_k3d_cluster target multicluster-test
"$bindir"/image-load --k3d ${images:+'--images'} target
fi
export context="k3d-source"
check_cluster
export context="k3d-target"
check_cluster
run_multicluster_test
exit_on_err "error calling 'run_multicluster_test'"
if [ -z "$skip_cluster_create" ]; then
delete_k3d_cluster source
delete_k3d_cluster target
else
export context="k3d-source"
cleanup_cluster
export context="k3d-target"
cleanup_cluster
fi
}
get_test_config() {
local name=$1
config=''
@ -345,9 +374,16 @@ run_uninstall_test() {
run_test "$test_directory/uninstall/uninstall_test.go" --uninstall=true
}
run_multicluster_test() {
export context="k3d-source"
run_test "$test_directory/install_test.go" --multicluster
export context="k3d-target"
run_test "$test_directory/install_test.go" --multicluster
}
run_deep_test() {
local tests=()
run_test "$test_directory/install_test.go" --multicluster
run_test "$test_directory/install_test.go"
while IFS= read -r line; do tests+=("$line"); done <<< "$(go list "$test_directory"/.../...)"
for test in "${tests[@]}"; do
run_test "$test"
@ -369,7 +405,7 @@ run_helm-deep_test() {
helm_multicluster_chart="$( cd "$bindir"/.. && pwd )"/charts/linkerd2-multicluster
run_test "$test_directory/install_test.go" --helm-path="$helm_path" --helm-chart="$helm_chart" \
--helm-release="$helm_release_name" --multicluster-helm-chart="$helm_multicluster_chart" \
--multicluster-helm-release="$helm_multicluster_release_name" --multicluster
--multicluster-helm-release="$helm_multicluster_release_name"
while IFS= read -r line; do tests+=("$line"); done <<< "$(go list "$test_directory"/.../...)"
for test in "${tests[@]}"; do
run_test "$test"
@ -378,12 +414,12 @@ run_helm-deep_test() {
}
run_external-issuer_test() {
run_test "$test_directory/install_test.go" --external-issuer=true --multicluster
run_test "$test_directory/install_test.go" --external-issuer=true
run_test "$test_directory/externalissuer/external_issuer_test.go" --external-issuer=true
}
run_cluster-domain_test() {
run_test "$test_directory/install_test.go" --cluster-domain='custom.domain' --multicluster
run_test "$test_directory/install_test.go" --cluster-domain='custom.domain'
}
# exit_on_err should be called right after a command to check the result status

102
bin/image-load Executable file
View File

@ -0,0 +1,102 @@
#!/usr/bin/env bash
set -eo pipefail
kind=""
k3d=""
images=""
while :
do
case $1 in
-h|--help)
echo "Load into KinD/k3d the images for Linkerd's proxy, controller, web, grafana, cli-bin, debug and cni-plugin."
echo ""
echo "Usage:"
echo " bin/image-load [--kind] [--k3d] [--images]"
echo ""
echo "Examples:"
echo ""
echo " # Load images from the local docker instance into KinD"
echo " bin/image-load"
echo ""
echo " # Load images from tar files located under the 'image-archives' directory into k3d"
echo " bin/image-load --k3d --images"
echo ""
echo "Available Commands:"
echo " --kind: use a KinD cluster (default)."
echo " --k3d: use a k3d cluster."
echo " --images: Load the images from local .tar files in the current directory."
exit 0
;;
--kind)
kind=1
;;
--k3d)
k3d=1
;;
--images)
images=1
;;
*)
break
esac
shift
done
bindir=$( cd "${0%/*}" && pwd )
if [ -n "$k3d" ]; then
if [ -n "$kind" ]; then
echo $k3d
echo "Error: --kind and --k3d can't be used simultaneously" >&2
exit 1
fi
cluster=${1:-"k3s-default"}
bin="$bindir"/k3d
image_sub_cmd=(image import -c "$cluster")
else
kind=1
cluster=${1:-"kind"}
bin="$bindir"/kind
if [ $images ]; then
image_sub_cmd=(load image-archive --name "$cluster")
else
image_sub_cmd=(load docker-image --name "$cluster")
fi
fi
if [ -z "$images" ]; then
# shellcheck source=_tag.sh
. "$bindir"/_tag.sh
# shellcheck source=_docker.sh
. "$bindir"/_docker.sh
TAG=${TAG:-$(head_root_tag)}
fi
# This is really to load the binary synchronously, before
# the parallel executions below attempt doing so
"$bin" version
rm -f load_fail
for img in proxy controller web grafana cli-bin debug cni-plugin ; do
if [ $images ]; then
param="image-archives/$img.tar"
else
param="$DOCKER_REGISTRY/$img:$TAG"
fi
if [ -n "$kind" ]; then
"$bin" "${image_sub_cmd[@]}" "${param[@]}" || touch load_fail &
else
# "k3d image import" commands don't behave well when parallelized
"$bin" "${image_sub_cmd[@]}" "${param[@]}"
fi
done
wait < <(jobs -p)
if [ -f load_fail ]; then
echo "Loading docker images into the cluster failed."
rm load_fail
exit 1
fi

32
bin/k3d Executable file
View File

@ -0,0 +1,32 @@
#!/usr/bin/env sh
set -eu
k3dversion=v3.0.2
bindir=$( cd "${0%/*}" && pwd )
targetbin=$( cd "$bindir"/.. && pwd )/target/bin
k3dbin=$targetbin/.k3d-$k3dversion
if [ ! -f "$k3dbin" ]; then
if [ "$(uname -s)" = Darwin ]; then
os=darwin
arch=amd64
elif [ "$(uname -o)" = Msys ]; then
os=windows
arch=amd64
else
os=linux
case $(uname -m) in
x86_64) arch=amd64 ;;
arm) arch=arm64 ;;
esac
fi
mkdir -p "$targetbin"
curl -sfL -o "$k3dbin" https://github.com/rancher/k3d/releases/download/$k3dversion/k3d-$os-$arch
chmod +x "$k3dbin"
fi
"$k3dbin" "$@"

93
bin/kind-load
View File

@ -1,93 +0,0 @@
#!/usr/bin/env bash
set -eo pipefail
images=""
images_host=""
while :
do
case $1 in
-h|--help)
echo "Load into KinD the images for Linkerd's proxy, controller, web, grafana, cli-bin, debug and cni-plugin."
echo ""
echo "Usage:"
echo " bin/kind-load [--images] [--images-host ssh://linkerd-docker]"
echo ""
echo "Examples:"
echo ""
echo " # Load images from the local docker instance"
echo " bin/kind-load"
echo ""
echo " # Load images from tar files located under the 'image-archives' directory"
echo " bin/kind-load --images"
echo ""
echo " # Retrieve images from a remote docker instance and then load them into KinD"
echo " bin/kind-load --images --images-host ssh://linkerd-docker"
echo ""
echo "Available Commands:"
echo " --images: use 'kind load image-archive' to load the images from local .tar files in the current directory."
echo " --images-host: the argument to this option is used as the remote docker instance from which images are first retrieved"
echo " (using 'docker save') to be then loaded into KinD. This command requires --images."
exit 0
;;
--images)
images=1
;;
--images-host)
images_host=$2
if [ -z "$images_host" ]; then
echo "Error: the argument for --images-host was not specified"
exit 1
fi
shift
;;
*)
break
esac
shift
done
if [ "$images_host" ] && [ -z "$images" ]; then
echo "Error: --images-host needs to be used with --images" >&2
exit 1
fi
# If cluster name is unset or null, default to "kind".
cluster=${1:-"kind"}
bindir=$( cd "${0%/*}" && pwd )
if [ -z "$images" ] || [ -n "$images_host" ]; then
# shellcheck source=_tag.sh
. "$bindir"/_tag.sh
# shellcheck source=_docker.sh
. "$bindir"/_docker.sh
TAG=${TAG:-$(head_root_tag)}
fi
# This is really to load the kind binary synchronously, before
# the parallel executions below attempt doing so
"$bindir"/kind version
rm -f load_fail
for img in proxy controller web grafana cli-bin debug cni-plugin ; do
if [ $images ]; then
if [ "$images_host" ]; then
mkdir -p image-archives
DOCKER_HOST=$images_host docker save "$DOCKER_REGISTRY/$img:$TAG" > image-archives/$img.tar
fi
cmd=(image-archive "image-archives/$img.tar")
else
cmd=(docker-image "$DOCKER_REGISTRY/$img:$TAG")
fi
"$bindir"/kind --name "$cluster" load "${cmd[@]}" || touch load_fail &
done
wait < <(jobs -p)
if [ -f load_fail ]; then
echo "Loading docker images into KinD cluster failed."
rm load_fail
exit 1
fi

39
test/integration/install_test.go
View File

@ -462,9 +462,6 @@ func TestControlPlaneResourcesPostInstall(t *testing.T) {
}
func TestInstallMulticluster(t *testing.T) {
if !TestHelper.Multicluster() {
return
}
if TestHelper.GetMulticlusterHelmReleaseName() != "" {
flags := []string{
"--set", "linkerdVersion=" + TestHelper.GetVersion(),
@ -474,7 +471,7 @@ func TestInstallMulticluster(t *testing.T) {
testutil.AnnotatedFatalf(t, "'helm install' command failed",
"'helm install' command failed\n%s\n%s", stdout, stderr)
}
} else {
} else if TestHelper.Multicluster() {
exec := append([]string{"multicluster"}, []string{
"install",
"--namespace", TestHelper.GetMulticlusterNamespace(),
@ -687,7 +684,7 @@ func testCheckCommand(t *testing.T, stage string, expectedVersion string, namesp
var golden string
if stage == "proxy" {
cmd = []string{"check", "--proxy", "--expected-version", expectedVersion, "--namespace", namespace, "--wait=0"}
if TestHelper.Multicluster() {
if TestHelper.GetMulticlusterHelmReleaseName() != "" || TestHelper.Multicluster() {
golden = "check.multicluster.proxy.golden"
} else if TestHelper.CNI() {
golden = "check.cni.proxy.golden"
@ -699,7 +696,7 @@ func testCheckCommand(t *testing.T, stage string, expectedVersion string, namesp
golden = "check.config.golden"
} else {
cmd = []string{"check", "--expected-version", expectedVersion, "--wait=0"}
if TestHelper.Multicluster() {
if TestHelper.GetMulticlusterHelmReleaseName() != "" || TestHelper.Multicluster() {
golden = "check.multicluster.golden"
} else if TestHelper.CNI() {
golden = "check.cni.golden"
@ -954,33 +951,3 @@ func TestRestarts(t *testing.T) {
}
}
}
//TODO Put that in test_cleanup when we have adequate resource labels
func TestUninstallMulticluster(t *testing.T) {
if !TestHelper.Multicluster() {
return
}
if TestHelper.GetMulticlusterHelmReleaseName() != "" {
if stdout, stderr, err := TestHelper.HelmUninstallMulticluster(TestHelper.GetMulticlusterHelmChart()); err != nil {
testutil.AnnotatedFatalf(t, "'helm delete' command failed",
"'helm delete' command failed\n%s\n%s", stdout, stderr)
}
} else {
exec := append([]string{"multicluster"}, []string{
"install",
"--namespace", TestHelper.GetMulticlusterNamespace(),
}...)
out, stderr, err := TestHelper.LinkerdRun(exec...)
if err != nil {
testutil.AnnotatedFatalf(t, "'linkerd multicluster install' command failed",
"'linkerd multicluster' command failed: \n%s\n%s", out, stderr)
}
out, err = TestHelper.Kubectl(out, []string{"delete", "-f", "-"}...)
if err != nil {
testutil.AnnotatedFatalf(t, "'kubectl delete' command failed",
"'kubectl apply' command failed\n%s", out)
}
}
}