mirror of https://github.com/kubernetes/kops.git
Merge pull request #13265 from jiahuif-forks/feature/leader-migration-tester
re-organize Leader Migration test with exec tester
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
610f0354ff
|
|
@ -20,7 +20,7 @@ set -o pipefail
|
|||
set -o xtrace
|
||||
|
||||
echo "CLOUD_PROVIDER=${CLOUD_PROVIDER}"
|
||||
echo "CLUSTER_NAME=${CLUSTER_NAME}"
|
||||
echo "CLUSTER_NAME=${CLUSTER_NAME-}"
|
||||
|
||||
if [[ -z "${WORKSPACE-}" ]]; then
|
||||
export WORKSPACE
|
||||
|
|
@ -43,7 +43,7 @@ export KOPS_FEATURE_FLAGS="SpecOverrideFlag"
|
|||
export KOPS_RUN_TOO_NEW_VERSION=1
|
||||
|
||||
if [[ -z "${DISCOVERY_STORE-}" ]]; then
|
||||
DISCOVERY_STORE="${KOPS_STATE_STORE}"
|
||||
DISCOVERY_STORE="${KOPS_STATE_STORE-}"
|
||||
fi
|
||||
|
||||
if [[ ${KOPS_IRSA-} = true ]]; then
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2021 The Kubernetes Authors.
|
||||
# Copyright 2022 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -19,17 +19,17 @@ SCENARIO_ROOT="${REPO_ROOT}/tests/e2e/scenarios/upgrade-ha-leader-migration"
|
|||
|
||||
source "${REPO_ROOT}"/tests/e2e/scenarios/lib/common.sh
|
||||
|
||||
OVERRIDES=("--node-count=1" "--master-count=3")
|
||||
OVERRIDES=("--channel=alpha" "--node-count=1" "--master-count=3")
|
||||
|
||||
case "${CLOUD_PROVIDER}" in
|
||||
gce)
|
||||
export KOPS_FEATURE_FLAGS=AlphaAllowGCE,SpecOverrideFlag
|
||||
OVERRIDES+=(
|
||||
"--zones=us-central1-a,us-central1-b,us-central1-c"
|
||||
"--master-zones=us-central1-a,us-central1-b,us-central1-c"
|
||||
"--gce-service-account=default" # see test-infra#24749
|
||||
)
|
||||
;;
|
||||
export KOPS_FEATURE_FLAGS=AlphaAllowGCE,SpecOverrideFlag
|
||||
OVERRIDES+=(
|
||||
"--zones=us-central1-a,us-central1-b,us-central1-c"
|
||||
"--master-zones=us-central1-a,us-central1-b,us-central1-c"
|
||||
"--gce-service-account=default" # see test-infra#24749
|
||||
)
|
||||
;;
|
||||
*) ;;
|
||||
|
||||
esac
|
||||
|
|
@ -39,73 +39,21 @@ kops-acquire-latest
|
|||
# the migration in this test case is KCM to KCM+CCM, which should happen
|
||||
# during the upgrade from 1.23 to 1.24
|
||||
K8S_VERSION_A=$(curl https://storage.googleapis.com/kubernetes-release/release/stable-1.23.txt)
|
||||
export K8S_VERSION_A
|
||||
K8S_VERSION_B=$(curl https://storage.googleapis.com/kubernetes-release/release/latest-1.24.txt)
|
||||
export K8S_VERSION_B
|
||||
|
||||
# spin up the 1.23 cluster
|
||||
OVERRIDES="${OVERRIDES[*]}" K8S_VERSION="${K8S_VERSION_A}" kops-up
|
||||
|
||||
# create the recorder pod
|
||||
kubectl create -f "${SCENARIO_ROOT}/resources.yaml"
|
||||
(cd "${SCENARIO_ROOT}/cmd/recorder/" && go build -o "${WORKSPACE}/recorder")
|
||||
kubectl wait --for=condition=ready pod/recorder
|
||||
kubectl cp "${WORKSPACE}/recorder" recorder:/tmp/recorder
|
||||
rm "${WORKSPACE}/recorder"
|
||||
kubectl exec recorder -- /usr/bin/env sh -c 'mv /tmp/recorder /usr/local/bin/recorder'
|
||||
|
||||
# prepare for the upgrade
|
||||
# workaround current state of node IPAM controller
|
||||
"${KOPS}" edit cluster \
|
||||
'--set=cluster.spec.kubeControllerManager.enableLeaderMigration=false' \
|
||||
'--set=cluster.spec.cloudControllerManager.enableLeaderMigration=true' \
|
||||
'--set=cluster.spec.cloudControllerManager.controllers=*' \
|
||||
'--set=cluster.spec.cloudControllerManager.controllers=-nodeipam' \
|
||||
"--set=cluster.spec.kubernetesVersion=${K8S_VERSION_B}"
|
||||
|
||||
# perform the upgrade
|
||||
"${KOPS}" update cluster
|
||||
"${KOPS}" update cluster --admin --yes
|
||||
"${KOPS}" update cluster
|
||||
|
||||
# perform the rolling upgrade, we only care about the control plane
|
||||
"${KOPS}" rolling-update cluster
|
||||
"${KOPS}" rolling-update cluster --yes --validation-timeout=30m --instance-group-roles=master
|
||||
|
||||
# check recorder status
|
||||
phase=$(kubectl get pod -o go-template="{{.status.phase}}" recorder)
|
||||
|
||||
# if the recorder fails, which means a conflict is detected, dump log and exit
|
||||
if [[ ! "$phase" == Running ]]; then
|
||||
kubectl logs recorder
|
||||
kubectl delete -f "${SCENARIO_ROOT}/resources.yaml" # clean up
|
||||
echo "upgrade failed"
|
||||
exit 1
|
||||
# install kubetest2-test-exec if needed
|
||||
if ! command -v kubetest2-tester-exec >/dev/null; then
|
||||
go install sigs.k8s.io/kubetest2/kubetest2-tester-exec@latest
|
||||
fi
|
||||
|
||||
# prepare for the rollback
|
||||
"${KOPS}" edit cluster \
|
||||
'--set=cluster.spec.kubeControllerManager.enableLeaderMigration=true' \
|
||||
'--unset=cluster.spec.cloudControllerManager' \
|
||||
"--set=cluster.spec.kubernetesVersion=${K8S_VERSION_A}"
|
||||
|
||||
# perform the rollback
|
||||
"${KOPS}" update cluster
|
||||
"${KOPS}" update cluster --admin --yes
|
||||
"${KOPS}" update cluster
|
||||
|
||||
# perform the rolling rollback of the control plane
|
||||
"${KOPS}" rolling-update cluster
|
||||
"${KOPS}" rolling-update cluster --yes --validation-timeout=30m --instance-group-roles=master
|
||||
|
||||
# dump recorder output
|
||||
kubectl logs recorder
|
||||
|
||||
# check recorder status, again
|
||||
phase=$(kubectl get pod -o go-template="{{.status.phase}}" recorder)
|
||||
|
||||
# clean up
|
||||
kubectl delete -f "${SCENARIO_ROOT}/resources.yaml"
|
||||
|
||||
if [[ ! "$phase" == Running ]]; then
|
||||
echo "rollback failed"
|
||||
exit 1
|
||||
fi
|
||||
# run the test with kubetest2
|
||||
${KUBETEST2} \
|
||||
--up \
|
||||
--test=exec \
|
||||
--kops-binary-path="${KOPS}" \
|
||||
--kubernetes-version="${K8S_VERSION_A}" \
|
||||
--create-args="${OVERRIDES[*]}" \
|
||||
-- \
|
||||
"${SCENARIO_ROOT}/test-leader-migration.sh"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2022 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script runs the Leader Migration test suite against an existing replicated
|
||||
# control plane that is created by kOps.
|
||||
|
||||
# Please use kubetest2-tester-exec to invoke this script to properly setup testing
|
||||
# environment, even though the tester is technically not required.
|
||||
|
||||
# Please ensure K8S_VERSION_A and K8S_VERSION_B are set and exported
|
||||
# so that the test will upgrade the cluster from ${K8S_VERSION_A} to ${K8S_VERSION_B}.
|
||||
|
||||
set -xe
|
||||
|
||||
REPO_ROOT=$(git rev-parse --show-toplevel)
|
||||
SCENARIO_ROOT="${REPO_ROOT}/tests/e2e/scenarios/upgrade-ha-leader-migration"
|
||||
|
||||
# create the recorder pod
|
||||
kubectl create -f "${SCENARIO_ROOT}/resources.yaml"
|
||||
(cd "${SCENARIO_ROOT}/cmd/recorder/" && go build -o "${WORKSPACE}/recorder")
|
||||
kubectl wait --for=condition=ready pod/recorder
|
||||
kubectl cp "${WORKSPACE}/recorder" recorder:/tmp/recorder
|
||||
rm "${WORKSPACE}/recorder"
|
||||
kubectl exec recorder -- /usr/bin/env sh -c 'mv /tmp/recorder /usr/local/bin/recorder'
|
||||
|
||||
# prepare for the upgrade
|
||||
# workaround current state of node IPAM controller
|
||||
"${KOPS}" edit cluster \
|
||||
'--set=cluster.spec.kubeControllerManager.enableLeaderMigration=false' \
|
||||
'--set=cluster.spec.cloudControllerManager.enableLeaderMigration=true' \
|
||||
'--set=cluster.spec.cloudControllerManager.controllers=*' \
|
||||
'--set=cluster.spec.cloudControllerManager.controllers=-nodeipam' \
|
||||
"--set=cluster.spec.kubernetesVersion=${K8S_VERSION_B}"
|
||||
|
||||
# perform the upgrade
|
||||
"${KOPS}" update cluster
|
||||
"${KOPS}" update cluster --admin --yes
|
||||
"${KOPS}" update cluster
|
||||
|
||||
# perform the rolling upgrade, we only care about the control plane
|
||||
"${KOPS}" rolling-update cluster
|
||||
"${KOPS}" rolling-update cluster --yes --validation-timeout=30m --instance-group-roles=master
|
||||
|
||||
# check recorder status
|
||||
phase=$(kubectl get pod -o go-template="{{.status.phase}}" recorder)
|
||||
|
||||
# if the recorder fails, which means a conflict is detected, dump log and exit
|
||||
if [[ ! "$phase" == Running ]]; then
|
||||
kubectl logs recorder
|
||||
kubectl delete -f "${SCENARIO_ROOT}/resources.yaml" # clean up
|
||||
echo "upgrade failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# prepare for the rollback
|
||||
"${KOPS}" edit cluster \
|
||||
'--set=cluster.spec.kubeControllerManager.enableLeaderMigration=true' \
|
||||
'--unset=cluster.spec.cloudControllerManager' \
|
||||
"--set=cluster.spec.kubernetesVersion=${K8S_VERSION_A}"
|
||||
|
||||
# perform the rollback
|
||||
"${KOPS}" update cluster
|
||||
"${KOPS}" update cluster --admin --yes
|
||||
"${KOPS}" update cluster
|
||||
|
||||
# perform the rolling rollback of the control plane
|
||||
"${KOPS}" rolling-update cluster
|
||||
"${KOPS}" rolling-update cluster --yes --validation-timeout=30m --instance-group-roles=master
|
||||
|
||||
# dump recorder output
|
||||
kubectl logs recorder
|
||||
|
||||
# check recorder status, again
|
||||
phase=$(kubectl get pod -o go-template="{{.status.phase}}" recorder)
|
||||
|
||||
# clean up
|
||||
kubectl delete -f "${SCENARIO_ROOT}/resources.yaml"
|
||||
|
||||
if [[ ! "$phase" == Running ]]; then
|
||||
echo "rollback failed"
|
||||
exit 1
|
||||
fi
|
||||
Loading…
Reference in New Issue