#!/usr/bin/env bash set -o errexit set -o nounset set -o pipefail # This script starts a local karmada control plane based on current codebase and with a certain number of clusters joined. REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. # This script intended used in following scenarios: source ${REPO_ROOT}/hack/util.sh # 1. used by developer to setup develop environment quickly. # 2. used by e2e testing to setup test environment automatically. REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source ${REPO_ROOT}/hack/util.sh KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"} MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"member1"} MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"member2"} HOST_CLUSTER_KUBECONFIG=${HOST_CLUSTER_KUBECONFIG:-"${KUBECONFIG_PATH}/karmada-host.config"} MEMBER_CLUSTER_1_KUBECONFIG=${MEMBER_CLUSTER_1_KUBECONFIG:-"${KUBECONFIG_PATH}/member1.config"} MEMBER_CLUSTER_2_KUBECONFIG=${MEMBER_CLUSTER_2_KUBECONFIG:-"${KUBECONFIG_PATH}/member2.config"} CLUSTER_VERSION=${CLUSTER_VERSION:-"kindest/node:v1.19.1"} CERT_DIR=${CERT_DIR:-"/var/run/karmada"} mkdir -p "${CERT_DIR}" &>/dev/null || sudo mkdir -p "${CERT_DIR}" KARMADA_APISERVER_CONFIG="${CERT_DIR}/karmada-apiserver.config" KARMADA_APISERVER_SECURE_PORT=${KARMADA_APISERVER_SECURE_PORT:-5443} CFSSL_VERSION="v1.5.0" CONTROLPLANE_SUDO=$(test -w "${CERT_DIR}" || echo "sudo -E") ROOT_CA_FILE=${CERT_DIR}/server-ca.crt KIND_LOG_FILE=${KIND_LOG_FILE:-"/tmp/karmada"} # generate a secret to store the certificates function generate_cert_secret { local karmada_crt_file=${CERT_DIR}/karmada.crt local karmada_key_file=${CERT_DIR}/karmada.key sudo chmod 0644 ${karmada_crt_file} sudo chmod 0644 ${karmada_key_file} local karmada_ca=$(sudo cat ${ROOT_CA_FILE} | base64 | tr "\n" " "|sed s/[[:space:]]//g) local karmada_crt=$(sudo cat ${karmada_crt_file} | base64 | tr "\n" " "|sed s/[[:space:]]//g) local karmada_key=$(sudo cat ${karmada_key_file} | base64 | tr "\n" " "|sed s/[[:space:]]//g) local TEMP_PATH=$(mktemp -d) cp -rf ${REPO_ROOT}/artifacts/deploy/karmada-cert-secret.yaml ${TEMP_PATH}/karmada-cert-secret-tmp.yaml cp -rf ${REPO_ROOT}/artifacts/deploy/secret.yaml ${TEMP_PATH}/secret-tmp.yaml cp -rf ${REPO_ROOT}/artifacts/deploy/karmada-webhook-cert-secret.yaml ${TEMP_PATH}/karmada-webhook-cert-secret-tmp.yaml sed -i "s/{{ca_crt}}/${karmada_ca}/g" ${TEMP_PATH}/karmada-cert-secret-tmp.yaml sed -i "s/{{client_cer}}/${karmada_crt}/g" ${TEMP_PATH}/karmada-cert-secret-tmp.yaml sed -i "s/{{client_key}}/${karmada_key}/g" ${TEMP_PATH}/karmada-cert-secret-tmp.yaml sed -i "s/{{ca_crt}}/${karmada_ca}/g" ${TEMP_PATH}/secret-tmp.yaml sed -i "s/{{client_cer}}/${karmada_crt}/g" ${TEMP_PATH}/secret-tmp.yaml sed -i "s/{{client_key}}/${karmada_key}/g" ${TEMP_PATH}/secret-tmp.yaml sed -i "s/{{server_key}}/${karmada_key}/g" ${TEMP_PATH}/karmada-webhook-cert-secret-tmp.yaml sed -i "s/{{server_certificate}}/${karmada_crt}/g" ${TEMP_PATH}/karmada-webhook-cert-secret-tmp.yaml kubectl apply -f ${TEMP_PATH}/karmada-cert-secret-tmp.yaml kubectl apply -f ${TEMP_PATH}/secret-tmp.yaml kubectl apply -f ${TEMP_PATH}/karmada-webhook-cert-secret-tmp.yaml rm -rf "${TEMP_PATH}" } function installCRDs() { if [[ ! -f ${KARMADA_APISERVER_CONFIG} ]]; then echo "Please provide kubeconfig to connect karmada apiserver" return 1 fi # install APIs kubectl apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/cluster.karmada.io_clusters.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/policy.karmada.io_propagationpolicies.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/policy.karmada.io_resourcebindings.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/policy.karmada.io_overridepolicies.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/work.karmada.io_works.yaml" } #step1. create host cluster and member clusters in parallel util::create_cluster ${HOST_CLUSTER_NAME} ${HOST_CLUSTER_KUBECONFIG} ${CLUSTER_VERSION} ${KIND_LOG_FILE} util::create_cluster ${MEMBER_CLUSTER_1_NAME} ${MEMBER_CLUSTER_1_KUBECONFIG} ${CLUSTER_VERSION} ${KIND_LOG_FILE} util::create_cluster ${MEMBER_CLUSTER_2_NAME} ${MEMBER_CLUSTER_2_KUBECONFIG} ${CLUSTER_VERSION} ${KIND_LOG_FILE} #step2. make images and get karmadactl export VERSION="latest" export REGISTRY="swr.ap-southeast-1.myhuaweicloud.com/karmada" make images GO111MODULE=on go install "github.com/karmada-io/karmada/cmd/karmadactl" GOPATH=$(go env | grep GOPATH | awk -F '=' '{print $2}'| sed 's/\"//g') KARMADACTL_BIN="${GOPATH}/bin/karmadactl" #step3. generate cert util::cmd_must_exist "openssl" util::cmd_must_exist_cfssl ${CFSSL_VERSION} # create CA signers util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" server '"client auth","server auth"' # signs a certificate util::create_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "server-ca" karmada system:admin kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" #step4. wait until the host cluster ready echo "Waiting for the host clusters to be ready..." util::check_clusters_ready ${HOST_CLUSTER_KUBECONFIG} ${HOST_CLUSTER_NAME} #step5. load components images to kind cluster kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --name="${HOST_CLUSTER_NAME}" kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}" kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}" #step6. generate kubeconfig and cert secret KARMADA_APISERVER_IP=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${HOST_CLUSTER_NAME}-control-plane") util::write_client_kubeconfig "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "${ROOT_CA_FILE}" "${KARMADA_APISERVER_IP}" "${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver #step7. install karmada control plane components export KUBECONFIG="${HOST_CLUSTER_KUBECONFIG}" # create namespace for control plane components kubectl apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml" # create service account, cluster role for controller-manager kubectl apply -f "${REPO_ROOT}/artifacts/deploy/serviceaccount.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/clusterrole.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/clusterrolebinding.yaml" generate_cert_secret # deploy karmada etcd kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-etcd.yaml" # Wait for karmada-etcd to come up before launching the rest of the components. util::wait_pod_ready ${ETCD_POD_LABEL} "karmada-system" # deploy karmada apiserver TEMP_PATH=$(mktemp -d) cp -rf ${REPO_ROOT}/artifacts/deploy/karmada-apiserver.yaml ${TEMP_PATH}/karmada-apiserver-tmp.yaml sed -i "s/{{api_addr}}/${KARMADA_APISERVER_IP}/g" ${TEMP_PATH}/karmada-apiserver-tmp.yaml kubectl apply -f "${TEMP_PATH}/karmada-apiserver-tmp.yaml" rm -rf "${TEMP_PATH}" # wait for karmada-apiserver to come up before launching the rest of the components. util::wait_pod_ready ${APISERVER_POD_LABEL} "karmada-system" # deploy kube controller manager kubectl apply -f "${REPO_ROOT}/artifacts/deploy/kube-controller-manager.yaml" # install CRD APIs on karmada apiserver. export KUBECONFIG=${KARMADA_APISERVER_CONFIG} installCRDs # deploy webhook configurations on karmada apiserver util::deploy_webhook_configuration ${ROOT_CA_FILE} "${REPO_ROOT}/artifacts/deploy/webhook-configuration.yaml" # deploy karmada-controller-manager on host cluster export KUBECONFIG=${HOST_CLUSTER_KUBECONFIG} kubectl apply -f "${REPO_ROOT}/artifacts/deploy/controller-manager.yaml" # deploy karmada-scheduler on host cluster kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-scheduler.yaml" # deploy karmada-webhook on host cluster kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-webhook.yaml" # make sure all karmada control plane components are ready util::wait_pod_ready ${KARMADA_CONTROLLER_LABEL} "karmada-system" util::wait_pod_ready ${KARMADA_SCHEDULER_LABEL} "karmada-system" util::wait_pod_ready ${KUBE_CONTROLLER_POD_LABEL} "karmada-system" util::wait_pod_ready ${KARMADA_WEBHOOK_LABEL} "karmada-system" # wait until the member cluster ready util::check_clusters_ready ${MEMBER_CLUSTER_1_KUBECONFIG} "member1" util::check_clusters_ready ${MEMBER_CLUSTER_2_KUBECONFIG} "member2" # join member clusters export KUBECONFIG=${KARMADA_APISERVER_CONFIG} ${KARMADACTL_BIN} join member1 --cluster-kubeconfig="${MEMBER_CLUSTER_1_KUBECONFIG}" ${KARMADACTL_BIN} join member2 --cluster-kubeconfig="${MEMBER_CLUSTER_2_KUBECONFIG}" function print_success() { echo echo "Local Karmada is running." echo "To start using your karmada, run:" cat <