Merge pull request #5519 from zhzhuang-zju/operator
add ci for karmada operator
This commit is contained in:
karmada-bot
GitHub
commit
e204459087
|
|
@ -0,0 +1,82 @@
|
|||
name: Operator
|
||||
on:
|
||||
# Run this workflow every time a new commit pushed to upstream/fork repository.
|
||||
# Run workflow on fork repository will help contributors find and resolve issues before sending a PR.
|
||||
push:
|
||||
# Exclude branches created by Dependabot to avoid triggering current workflow
|
||||
# for PRs initiated by Dependabot.
|
||||
branches-ignore:
|
||||
- 'dependabot/**'
|
||||
pull_request:
|
||||
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
permissions:
|
||||
contents: read # for actions/checkout to fetch code
|
||||
jobs:
|
||||
test-on-kubernetes-matrix:
|
||||
name: Test on Kubernetes
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# Here support the latest three minor releases of Kubernetes, this can be considered to be roughly
|
||||
# the same as the End of Life of the Kubernetes release: https://kubernetes.io/releases/
|
||||
# Please remember to update the CI Schedule Workflow when we add a new version.
|
||||
k8s: [ v1.29.0, v1.30.0, v1.31.0 ]
|
||||
steps:
|
||||
# Free up disk space on Ubuntu
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
# this might remove tools that are actually needed, if set to "true" but frees about 6 GB
|
||||
tool-cache: false
|
||||
# all of these default to true, but feel free to set to "false" if necessary for your workflow
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
docker-images: false
|
||||
swap-storage: false
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. 0 indicates all history for all branches and tags.
|
||||
# We need to guess version via git tags.
|
||||
fetch-depth: 0
|
||||
- name: install Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: setup operator test environment
|
||||
run: |
|
||||
export CLUSTER_VERSION=kindest/node:${{ matrix.k8s }}
|
||||
hack/local-up-karmada-by-operator.sh
|
||||
- name: run operator test
|
||||
run: |
|
||||
# run a single e2e
|
||||
export KUBECONFIG=${HOME}/.kube/karmada.config
|
||||
kubectl config use-context karmada-apiserver
|
||||
GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo
|
||||
ginkgo -v --race --trace -p --focus="[BasicPropagation] propagation testing deployment propagation testing" ./test/e2e/
|
||||
- name: export logs
|
||||
if: always()
|
||||
run: |
|
||||
export ARTIFACTS_PATH=${{ github.workspace }}/karmada-operator-test-logs/${{ matrix.k8s }}/
|
||||
mkdir -p $ARTIFACTS_PATH
|
||||
|
||||
mkdir -p $ARTIFACTS_PATH/karmada-host
|
||||
kind export logs --name=karmada-host $ARTIFACTS_PATH/karmada-host
|
||||
- name: upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: karmada_operator_test_logs_${{ matrix.k8s }}
|
||||
path: ${{ github.workspace }}/karmada-operator-test-logs/${{ matrix.k8s }}/
|
||||
- name: upload kind logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: karmada_kind_log_${{ matrix.k8s }}
|
||||
path: /tmp/karmada/
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
#!/usr/bin/env bash
|
||||
# Copyright 2024 The Karmada Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
|
||||
# This script deploy karmada instance to any cluster you want via karmada-operator.
|
||||
# This script depends on utils in: ${REPO_ROOT}/hack/util.sh
|
||||
|
||||
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
|
||||
source ${REPO_ROOT}/hack/util.sh
|
||||
|
||||
KARMADA_SYSTEM_NAMESPACE="karmada-system"
|
||||
KARMADA_INSTANCE_NAME=${KARMADA_INSTANCE_NAME:-"karmada-demo"}
|
||||
KARMADA_INSTANCE_NAMESPACE=${KARMADA_INSTANCE_NAMESPACE:-"test"}
|
||||
|
||||
CERT_DIR=${CERT_DIR:-"${HOME}/.karmada"}
|
||||
mkdir -p "${CERT_DIR}" &>/dev/null || mkdir -p "${CERT_DIR}"
|
||||
rm -f "${CERT_DIR}/*" &>/dev/null || rm -f "${CERT_DIR}/*"
|
||||
|
||||
function usage() {
|
||||
echo "This script deploys karmada instance to a given cluster via karmada-operator."
|
||||
echo "Note: This script is an internal script and is not intended used by end-users."
|
||||
echo "Usage: hack/deploy-karmada-by-operator.sh <KUBECONFIG> <CONTEXT_NAME> <KARMADA_CONTEXT_NAME> <KARMADA_IMAGE_TAG> <ADDON_NEEDED> <CRD_DOWNLOAD_URL>"
|
||||
echo "Example: hack/deploy-karmada-by-operator.sh ~/.kube/members.config member1 karmada-apiserver v1.11.0 true https://github.com/karmada-io/karmada/releases/download/v1.11.0/crds.tar.gz"
|
||||
echo -e "Parameters:\n\tKUBECONFIG\t\tYour cluster's kubeconfig that you want to install to"
|
||||
echo -e "\tCONTEXT_NAME\t\tThe name of context in 'kubeconfig'"
|
||||
echo -e "\tKARMADA_CONTEXT_NAME\t\tThe context name of karmada instance, and different Karmada instances must have unique contexts to avoid being overwritten.'"
|
||||
echo -e "\tKARMADA_IMAGE_TAG\t\tThe tag of image'"
|
||||
echo -e "\tADDON_NEEDED\t\tWhether you need to install addons(KarmadaSearch&karmadaDescheduler), optional, defaults to false."
|
||||
echo -e "\tCRD_DOWNLOAD_URL\t\tThe download url for CRDs, optional.'"
|
||||
}
|
||||
|
||||
if [[ $# -le 4 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check config file existence
|
||||
HOST_CLUSTER_KUBECONFIG=$1
|
||||
if [[ ! -f "${HOST_CLUSTER_KUBECONFIG}" ]]; then
|
||||
echo -e "ERROR: failed to get kubernetes config file: '${HOST_CLUSTER_KUBECONFIG}', not existed.\n"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check context existence
|
||||
CONTEXT_NAME=$2
|
||||
if ! kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" config get-contexts "${CONTEXT_NAME}" > /dev/null 2>&1;
|
||||
then
|
||||
echo -e "ERROR: failed to get context: '${CONTEXT_NAME}' not in ${HOST_CLUSTER_KUBECONFIG}. \n"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check for duplicate karmada context name.
|
||||
KARMADA_CONTEXT_NAME=$3
|
||||
if kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" config get-contexts "${KARMADA_CONTEXT_NAME}" > /dev/null 2>&1;
|
||||
then
|
||||
echo -e "ERROR: context: '${KARMADA_CONTEXT_NAME}' already exists in ${HOST_CLUSTER_KUBECONFIG}. \n"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TEMP_PATH_BOOTSTRAP=$(mktemp -d)
|
||||
trap '{ rm -rf ${TEMP_PATH_BOOTSTRAP}; }' EXIT
|
||||
cp -rf "${REPO_ROOT}"/operator/config/samples/karmada-sample.yaml "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
|
||||
|
||||
if kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" get namespace ${KARMADA_INSTANCE_NAMESPACE} > /dev/null 2>&1; then
|
||||
echo "Namespace '${KARMADA_INSTANCE_NAMESPACE}' already exists."
|
||||
else
|
||||
echo "Namespace '${KARMADA_INSTANCE_NAMESPACE}' does not exist. Creating now..."
|
||||
kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" create ns ${KARMADA_INSTANCE_NAMESPACE}
|
||||
fi
|
||||
|
||||
# modify `karmada-sample.yaml` based on custom configuration.
|
||||
ADDON_NEEDED=${5:-false}
|
||||
# if choosing install addons, append karmadaSearch and karmadaDescheduler to 'karmada-sample.yaml'
|
||||
if [ ${ADDON_NEEDED} ]; then
|
||||
echo -e ' karmadaDescheduler:\n imageRepository: docker.io/karmada/karmada-descheduler\n imageTag: {{image_tag}}\n replicas: 1' >> "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
|
||||
echo -e ' karmadaSearch:\n imageRepository: docker.io/karmada/karmada-search\n imageTag: {{image_tag}}\n replicas: 1' >> "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
|
||||
fi
|
||||
|
||||
IMAGE_TAG=$4
|
||||
sed -i'' -e "s/{{image_tag}}/${IMAGE_TAG}/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
|
||||
sed -i'' -e "s/{{karmada_instance_name}}/${KARMADA_INSTANCE_NAME}/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
|
||||
sed -i'' -e "s/{{karmada_instance_namespace}}/${KARMADA_INSTANCE_NAMESPACE}/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
|
||||
|
||||
CRD_DOWNLOAD_URL=${6:-""}
|
||||
if [[ -z ${CRD_DOWNLOAD_URL} ]]; then
|
||||
sed -i'' -e "s/{{crd_tarball}}/""/g" "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
|
||||
else
|
||||
CRD_TAR_BALL="\n httpSource:\n url: ${CRD_DOWNLOAD_URL}"
|
||||
awk -v pattern="{{crd_tarball}}" -v replacement="${CRD_TAR_BALL}" '{ gsub(pattern, replacement); print }' "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml > "${TEMP_PATH_BOOTSTRAP}"/temp && mv "${TEMP_PATH_BOOTSTRAP}"/temp "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
|
||||
fi
|
||||
|
||||
# create and wait for karmada instance to be ready
|
||||
kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${TEMP_PATH_BOOTSTRAP}"/karmada-sample-tmp.yaml
|
||||
kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" wait --for=condition=Ready --timeout=1000s karmada ${KARMADA_INSTANCE_NAME} -n ${KARMADA_INSTANCE_NAMESPACE}
|
||||
|
||||
# generate kubeconfig for karmada instance
|
||||
kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${CONTEXT_NAME}" get secret -n ${KARMADA_INSTANCE_NAMESPACE} ${KARMADA_INSTANCE_NAME}-admin-config -o jsonpath={.data.kubeconfig} | base64 -d > ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config
|
||||
cat ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config| grep "certificate-authority-data"| awk '{print $2}'| base64 -d > ${CERT_DIR}/ca.crt
|
||||
cat ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config| grep "client-certificate-data"| awk '{print $2}'| base64 -d > ${CERT_DIR}/karmada.crt
|
||||
cat ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config| grep "client-key-data"| awk '{print $2}'| base64 -d > ${CERT_DIR}/karmada.key
|
||||
KARMADA_APISERVER=$(cat ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config| grep "server:"| awk '{print $2}')
|
||||
|
||||
# write karmada api server config to kubeconfig file
|
||||
util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/ca.crt" "${CERT_DIR}/karmada.crt" "${CERT_DIR}/karmada.key" "${KARMADA_APISERVER}" ${KARMADA_CONTEXT_NAME}
|
||||
rm ~/.kube/${KARMADA_INSTANCE_NAME}-${KARMADA_INSTANCE_NAMESPACE}-tmp-apiserver.config
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
#!/usr/bin/env bash
|
||||
# Copyright 2024 The Karmada Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
|
||||
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
|
||||
source ${REPO_ROOT}/hack/util.sh
|
||||
KARMADA_SYSTEM_NAMESPACE="karmada-system"
|
||||
|
||||
function usage() {
|
||||
echo "This script will deploy karmada-operator on the specified cluster"
|
||||
echo "Usage: hack/deploy-karmada-operator.sh <KUBECONFIG> <CONTEXT_NAME>"
|
||||
echo "Example: hack/deploy-karmada-operator.sh ~/.kube/config karmada-host"
|
||||
}
|
||||
|
||||
if [[ $# -ne 2 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check kube config file existence
|
||||
if [[ ! -f "${1}" ]]; then
|
||||
echo -e "ERROR: failed to get kubernetes config file: '${1}', not existed.\n"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
KUBECONFIG=$1
|
||||
|
||||
# check context existence
|
||||
if ! kubectl config get-contexts "${2}" --kubeconfig="${KUBECONFIG}" > /dev/null 2>&1;
|
||||
then
|
||||
echo -e "ERROR: failed to get context: '${2}' not in ${KUBECONFIG}. \n"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
CONTEXT_NAME=$2
|
||||
|
||||
# make images
|
||||
export VERSION="latest"
|
||||
export REGISTRY="docker.io/karmada"
|
||||
make image-karmada-operator GOOS="linux" --directory=.
|
||||
|
||||
# load the karmada-operator images
|
||||
kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${CONTEXT_NAME}"
|
||||
|
||||
# create namespace `karmada-system`
|
||||
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml"
|
||||
|
||||
# install Karmada operator crds
|
||||
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f operator/config/crds/
|
||||
|
||||
# deploy karmada-operator
|
||||
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator-clusterrole.yaml"
|
||||
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator-clusterrolebinding.yaml"
|
||||
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator-serviceaccount.yaml"
|
||||
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator-deployment.yaml"
|
||||
|
||||
# wait karmada-operator ready
|
||||
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" wait --for=condition=Ready --timeout=30s pods -l karmada-app=karmada-operator -n ${KARMADA_SYSTEM_NAMESPACE}
|
||||
|
|
@ -265,7 +265,7 @@ else
|
|||
fi
|
||||
|
||||
# write karmada api server config to kubeconfig file
|
||||
util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${ROOT_CA_FILE}" "${CERT_DIR}/client.crt" "${CERT_DIR}/client.key" "${KARMADA_APISERVER_IP}" "${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver
|
||||
util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${ROOT_CA_FILE}" "${CERT_DIR}/client.crt" "${CERT_DIR}/client.key" "https://${KARMADA_APISERVER_IP}:${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver
|
||||
|
||||
# deploy kube controller manager
|
||||
cp "${REPO_ROOT}"/artifacts/deploy/kube-controller-manager.yaml "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml
|
||||
|
|
|
|||
|
|
@ -0,0 +1,223 @@
|
|||
#!/usr/bin/env bash
|
||||
# Copyright 2024 The Karmada Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# This script is used in workflow to validate karmada installation by operator.
|
||||
# It starts a local karmada control plane based on current codebase via karmada-operator and with a certain number of clusters joined.
|
||||
# This script depends on utils in: ${REPO_ROOT}/hack/util.sh
|
||||
# 1. used by developer to setup develop environment quickly.
|
||||
# 2. used by e2e testing to test if the operator installs correctly.
|
||||
|
||||
function usage() {
|
||||
echo "Usage:"
|
||||
echo " hack/local-up-karmada-by-operator.sh [-h]"
|
||||
echo " h: print help information"
|
||||
}
|
||||
|
||||
function getCrdsDir() {
|
||||
local path=$1
|
||||
local url=$2
|
||||
local key=$(echo "$url" | xargs) # Trim whitespace using xargs
|
||||
local hash=$(echo -n "$key" | sha256sum | awk '{print $1}') # Calculate SHA256 hash
|
||||
local hashedKey=${hash:0:64} # Take the first 64 characters of the hash
|
||||
echo "${path}/cache/${hashedKey}"
|
||||
}
|
||||
|
||||
while getopts 'h' OPT; do
|
||||
case $OPT in
|
||||
h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
?)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
|
||||
source "${REPO_ROOT}"/hack/util.sh
|
||||
|
||||
# variable define
|
||||
KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
|
||||
MAIN_KUBECONFIG=${MAIN_KUBECONFIG:-"${KUBECONFIG_PATH}/karmada.config"}
|
||||
HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"}
|
||||
KARMADA_APISERVER_CLUSTER_NAME=${KARMADA_APISERVER_CLUSTER_NAME:-"karmada-apiserver"}
|
||||
MEMBER_CLUSTER_KUBECONFIG=${MEMBER_CLUSTER_KUBECONFIG:-"${KUBECONFIG_PATH}/members.config"}
|
||||
MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"member1"}
|
||||
MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"member2"}
|
||||
PULL_MODE_CLUSTER_NAME=${PULL_MODE_CLUSTER_NAME:-"member3"}
|
||||
MEMBER_TMP_CONFIG_PREFIX="member-tmp"
|
||||
MEMBER_CLUSTER_1_TMP_CONFIG="${KUBECONFIG_PATH}/${MEMBER_TMP_CONFIG_PREFIX}-${MEMBER_CLUSTER_1_NAME}.config"
|
||||
MEMBER_CLUSTER_2_TMP_CONFIG="${KUBECONFIG_PATH}/${MEMBER_TMP_CONFIG_PREFIX}-${MEMBER_CLUSTER_2_NAME}.config"
|
||||
PULL_MODE_CLUSTER_TMP_CONFIG="${KUBECONFIG_PATH}/${MEMBER_TMP_CONFIG_PREFIX}-${PULL_MODE_CLUSTER_NAME}.config"
|
||||
CLUSTER_VERSION=${CLUSTER_VERSION:-"${DEFAULT_CLUSTER_VERSION}"}
|
||||
KIND_LOG_FILE=${KIND_LOG_FILE:-"/tmp/karmada"}
|
||||
KARMADA_SYSTEM_NAMESPACE="karmada-system"
|
||||
KARMADA_INSTANCE_NAME=${KARMADA_INSTANCE_NAME:-"karmada-demo"}
|
||||
KARMADA_INSTANCE_NAMESPACE=${KARMADA_INSTANCE_NAMESPACE:-"test"}
|
||||
|
||||
#step0: prepare
|
||||
# proxy setting in China mainland
|
||||
if [[ -n ${CHINA_MAINLAND:-} ]]; then
|
||||
util::set_mirror_registry_for_china_mainland ${REPO_ROOT}
|
||||
fi
|
||||
|
||||
# make sure go exists and the go version is a viable version.
|
||||
util::cmd_must_exist "go"
|
||||
util::verify_go_version
|
||||
|
||||
# make sure docker exists
|
||||
util::cmd_must_exist "docker"
|
||||
|
||||
# install kind and kubectl
|
||||
kind_version=v0.24.0
|
||||
echo -n "Preparing: 'kind' existence check - "
|
||||
if util::cmd_exist kind; then
|
||||
echo "passed"
|
||||
else
|
||||
echo "not pass"
|
||||
util::install_tools "sigs.k8s.io/kind" $kind_version
|
||||
fi
|
||||
|
||||
# get arch name and os name in bootstrap
|
||||
BS_ARCH=$(go env GOARCH)
|
||||
BS_OS=$(go env GOOS)
|
||||
# check arch and os name before installing
|
||||
util::install_environment_check "${BS_ARCH}" "${BS_OS}"
|
||||
echo -n "Preparing: 'kubectl' existence check - "
|
||||
if util::cmd_exist kubectl; then
|
||||
echo "passed"
|
||||
else
|
||||
echo "not pass"
|
||||
util::install_kubectl "" "${BS_ARCH}" "${BS_OS}"
|
||||
fi
|
||||
|
||||
#step1. create host cluster and member clusters in parallel
|
||||
#prepare for kindClusterConfig
|
||||
util::delete_necessary_resources "${MAIN_KUBECONFIG},${MEMBER_CLUSTER_KUBECONFIG}" "${HOST_CLUSTER_NAME},${MEMBER_CLUSTER_1_NAME},${MEMBER_CLUSTER_2_NAME},${PULL_MODE_CLUSTER_NAME}" "${KIND_LOG_FILE}"
|
||||
|
||||
util::create_cluster "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}"
|
||||
util::create_cluster "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}"/artifacts/kindClusterConfig/member1.yaml
|
||||
util::create_cluster "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}"/artifacts/kindClusterConfig/member2.yaml
|
||||
util::create_cluster "${PULL_MODE_CLUSTER_NAME}" "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}"/artifacts/kindClusterConfig/member3.yaml
|
||||
|
||||
#step2. make images and get karmadactl
|
||||
export VERSION="latest"
|
||||
export REGISTRY="docker.io/karmada"
|
||||
export KARMADA_IMAGE_LABEL_VALUE="May_be_pruned_in_local-up-karmada_by_operator.sh"
|
||||
export DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS:-} --label=image.karmada.io=${KARMADA_IMAGE_LABEL_VALUE}"
|
||||
make images GOOS="linux" --directory="${REPO_ROOT}"
|
||||
#clean up dangling images
|
||||
docker image prune --force --filter "label=image.karmada.io=${KARMADA_IMAGE_LABEL_VALUE}"
|
||||
|
||||
GO111MODULE=on go install "github.com/karmada-io/karmada/cmd/karmadactl"
|
||||
GOPATH=$(go env GOPATH | awk -F ':' '{print $1}')
|
||||
KARMADACTL_BIN="${GOPATH}/bin/karmadactl"
|
||||
|
||||
#step3. wait until the host cluster ready
|
||||
echo "Waiting for the host clusters to be ready..."
|
||||
util::check_clusters_ready "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}"
|
||||
|
||||
# load components images to host cluster
|
||||
kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${HOST_CLUSTER_NAME}"
|
||||
kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}"
|
||||
kind load docker-image "${REGISTRY}/karmada-descheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}"
|
||||
kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}"
|
||||
kind load docker-image "${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" --name="${HOST_CLUSTER_NAME}"
|
||||
kind load docker-image "${REGISTRY}/karmada-search:${VERSION}" --name="${HOST_CLUSTER_NAME}"
|
||||
kind load docker-image "${REGISTRY}/karmada-metrics-adapter:${VERSION}" --name="${HOST_CLUSTER_NAME}"
|
||||
|
||||
#step4. deploy karmada-operator
|
||||
"${REPO_ROOT}"/hack/deploy-karmada-operator.sh "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}"
|
||||
|
||||
#step5. install karmada instance by karmada-operator
|
||||
# prepare the local crds
|
||||
echo "Prepare the local crds"
|
||||
cd ${REPO_ROOT}/charts/karmada/
|
||||
cp -r _crds crds
|
||||
tar -zcvf ../../crds.tar.gz crds
|
||||
cd -
|
||||
|
||||
# copy the local crds.tar.gz file to the specified path of the karmada-operator, so that the karmada-operator will skip the step of downloading CRDs.
|
||||
CRDTARBALL_URL="local"
|
||||
DATA_DIR="/var/lib/karmada"
|
||||
CRD_CACHE_DIR=$(getCrdsDir "${DATA_DIR}" "${CRDTARBALL_URL}")
|
||||
OPERATOR_POD_NAME=$(kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" get pods -n ${KARMADA_SYSTEM_NAMESPACE} -l karmada-app=karmada-operator -o custom-columns=NAME:.metadata.name --no-headers)
|
||||
kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" exec -i ${OPERATOR_POD_NAME} -n ${KARMADA_SYSTEM_NAMESPACE} -- mkdir -p ${CRD_CACHE_DIR}
|
||||
kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" cp ${REPO_ROOT}/crds.tar.gz ${KARMADA_SYSTEM_NAMESPACE}/${OPERATOR_POD_NAME}:${CRD_CACHE_DIR}
|
||||
|
||||
# install karmada instance
|
||||
"${REPO_ROOT}"/hack/deploy-karmada-by-operator.sh "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" "${KARMADA_APISERVER_CLUSTER_NAME}" "${VERSION}" true "${CRDTARBALL_URL}"
|
||||
|
||||
#step6. join member clusters
|
||||
# wait until member clusters ready
|
||||
util::check_clusters_ready "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}"
|
||||
util::check_clusters_ready "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}"
|
||||
# connecting networks between karmada-host, member1 and member2 clusters
|
||||
echo "connecting cluster networks..."
|
||||
util::add_routes "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}"
|
||||
util::add_routes "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}"
|
||||
|
||||
util::add_routes "${HOST_CLUSTER_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}"
|
||||
util::add_routes "${MEMBER_CLUSTER_1_NAME}" "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}"
|
||||
|
||||
util::add_routes "${HOST_CLUSTER_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}"
|
||||
util::add_routes "${MEMBER_CLUSTER_2_NAME}" "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}"
|
||||
echo "cluster networks connected"
|
||||
|
||||
#join push mode member clusters
|
||||
export KUBECONFIG="${MAIN_KUBECONFIG}"
|
||||
${KARMADACTL_BIN} join --karmada-context="${KARMADA_APISERVER_CLUSTER_NAME}" ${MEMBER_CLUSTER_1_NAME} --cluster-kubeconfig="${MEMBER_CLUSTER_1_TMP_CONFIG}" --cluster-context="${MEMBER_CLUSTER_1_NAME}"
|
||||
${KARMADACTL_BIN} join --karmada-context="${KARMADA_APISERVER_CLUSTER_NAME}" ${MEMBER_CLUSTER_2_NAME} --cluster-kubeconfig="${MEMBER_CLUSTER_2_TMP_CONFIG}" --cluster-context="${MEMBER_CLUSTER_2_NAME}"
|
||||
|
||||
# wait until the pull mode cluster ready
|
||||
util::check_clusters_ready "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${PULL_MODE_CLUSTER_NAME}"
|
||||
kind load docker-image "${REGISTRY}/karmada-agent:${VERSION}" --name="${PULL_MODE_CLUSTER_NAME}"
|
||||
|
||||
#step7. deploy karmada agent in pull mode member clusters
|
||||
"${REPO_ROOT}"/hack/deploy-karmada-agent.sh "${MAIN_KUBECONFIG}" "${KARMADA_APISERVER_CLUSTER_NAME}" "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${PULL_MODE_CLUSTER_NAME}"
|
||||
|
||||
#step8. deploy metrics-server in member clusters
|
||||
"${REPO_ROOT}"/hack/deploy-k8s-metrics-server.sh "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${MEMBER_CLUSTER_1_NAME}"
|
||||
"${REPO_ROOT}"/hack/deploy-k8s-metrics-server.sh "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${MEMBER_CLUSTER_2_NAME}"
|
||||
"${REPO_ROOT}"/hack/deploy-k8s-metrics-server.sh "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${PULL_MODE_CLUSTER_NAME}"
|
||||
|
||||
# wait all of clusters member1, member2 and member3 status is ready
|
||||
util:wait_cluster_ready "${KARMADA_APISERVER_CLUSTER_NAME}" "${MEMBER_CLUSTER_1_NAME}"
|
||||
util:wait_cluster_ready "${KARMADA_APISERVER_CLUSTER_NAME}" "${MEMBER_CLUSTER_2_NAME}"
|
||||
util:wait_cluster_ready "${KARMADA_APISERVER_CLUSTER_NAME}" "${PULL_MODE_CLUSTER_NAME}"
|
||||
|
||||
#step9. merge temporary kubeconfig of member clusters by kubectl
|
||||
export KUBECONFIG=$(find ${KUBECONFIG_PATH} -maxdepth 1 -type f | grep ${MEMBER_TMP_CONFIG_PREFIX} | tr '\n' ':')
|
||||
kubectl config view --flatten > ${MEMBER_CLUSTER_KUBECONFIG}
|
||||
rm $(find ${KUBECONFIG_PATH} -maxdepth 1 -type f | grep ${MEMBER_TMP_CONFIG_PREFIX})
|
||||
|
||||
function print_success() {
|
||||
echo -e "$KARMADA_GREETING"
|
||||
echo "Local Karmada is running."
|
||||
echo -e "\nTo start using your karmada, run:"
|
||||
echo -e " export KUBECONFIG=${MAIN_KUBECONFIG}"
|
||||
echo "Please use 'kubectl config use-context ${HOST_CLUSTER_NAME}/${KARMADA_APISERVER_CLUSTER_NAME}' to switch the host and control plane cluster."
|
||||
echo -e "\nTo manage your member clusters, run:"
|
||||
echo -e " export KUBECONFIG=${MEMBER_CLUSTER_KUBECONFIG}"
|
||||
echo "Please use 'kubectl config use-context ${MEMBER_CLUSTER_1_NAME}/${MEMBER_CLUSTER_2_NAME}/${PULL_MODE_CLUSTER_NAME}' to switch to the different member cluster."
|
||||
}
|
||||
|
||||
print_success
|
||||
|
|
@ -261,11 +261,10 @@ function util::append_client_kubeconfig {
|
|||
local ca_file=$2
|
||||
local client_certificate_file=$3
|
||||
local client_key_file=$4
|
||||
local api_host=$5
|
||||
local api_port=$6
|
||||
local client_id=$7
|
||||
local token=${8:-}
|
||||
kubectl config set-cluster "${client_id}" --server=https://"${api_host}:${api_port}" --embed-certs --certificate-authority="${ca_file}" --kubeconfig="${kubeconfig_path}"
|
||||
local server=$5
|
||||
local client_id=$6
|
||||
local token=${7:-}
|
||||
kubectl config set-cluster "${client_id}" --server="${server}" --embed-certs --certificate-authority="${ca_file}" --kubeconfig="${kubeconfig_path}"
|
||||
kubectl config set-credentials "${client_id}" --token="${token}" --client-certificate="${client_certificate_file}" --client-key="${client_key_file}" --embed-certs=true --kubeconfig="${kubeconfig_path}"
|
||||
kubectl config set-context "${client_id}" --cluster="${client_id}" --user="${client_id}" --kubeconfig="${kubeconfig_path}"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
# used by hack/deploy-karmada-by-operator.sh to deploy a karmada instance
|
||||
apiVersion: operator.karmada.io/v1alpha1
|
||||
kind: Karmada
|
||||
metadata:
|
||||
name: {{karmada_instance_name}}
|
||||
namespace: {{karmada_instance_namespace}}
|
||||
spec:
|
||||
hostCluster:
|
||||
networking:
|
||||
dnsDomain: cluster.local
|
||||
crdTarball: {{crd_tarball}}
|
||||
components:
|
||||
etcd:
|
||||
local:
|
||||
imageRepository: registry.k8s.io/etcd
|
||||
imageTag: 3.5.13-0
|
||||
replicas: 1
|
||||
volumeData:
|
||||
# hostPath:
|
||||
# type: DirectoryOrCreate
|
||||
# path: /var/lib/karmada/etcd/karmada-demo
|
||||
volumeClaim:
|
||||
metadata:
|
||||
name: etcd-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 3Gi
|
||||
karmadaAPIServer:
|
||||
imageRepository: registry.k8s.io/kube-apiserver
|
||||
imageTag: v1.30.4
|
||||
replicas: 1
|
||||
serviceType: NodePort
|
||||
serviceSubnet: 10.96.0.0/12
|
||||
karmadaAggregatedAPIServer:
|
||||
imageRepository: docker.io/karmada/karmada-aggregated-apiserver
|
||||
imageTag: {{image_tag}}
|
||||
replicas: 1
|
||||
karmadaControllerManager:
|
||||
imageRepository: docker.io/karmada/karmada-controller-manager
|
||||
imageTag: {{image_tag}}
|
||||
replicas: 1
|
||||
karmadaScheduler:
|
||||
imageRepository: docker.io/karmada/karmada-scheduler
|
||||
imageTag: {{image_tag}}
|
||||
replicas: 1
|
||||
karmadaWebhook:
|
||||
imageRepository: docker.io/karmada/karmada-webhook
|
||||
imageTag: {{image_tag}}
|
||||
replicas: 1
|
||||
kubeControllerManager:
|
||||
imageRepository: registry.k8s.io/kube-controller-manager
|
||||
imageTag: v1.30.4
|
||||
replicas: 1
|
||||
karmadaMetricsAdapter:
|
||||
imageRepository: docker.io/karmada/karmada-metrics-adapter
|
||||
imageTag: {{image_tag}}
|
||||
replicas: 2
|
||||
Loading…
Reference in New Issue