diff --git a/artifacts/kindClusterConfig/karmada-host.yaml b/artifacts/kindClusterConfig/karmada-host.yaml new file mode 100644 index 000000000..1342653a5 --- /dev/null +++ b/artifacts/kindClusterConfig/karmada-host.yaml @@ -0,0 +1,11 @@ +kind: Cluster +apiVersion: "kind.x-k8s.io/v1alpha4" +networking: + apiServerAddress: "{{host_ipaddress}}" +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 5443 + hostPort: 5443 + protocol: TCP + listenAddress: "{{host_ipaddress}}" diff --git a/hack/create-cluster.sh b/hack/create-cluster.sh index 62fe5bfa9..7d2496645 100755 --- a/hack/create-cluster.sh +++ b/hack/create-cluster.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# This script only fits for Linux, macOS adaptation will come soon set -o errexit set -o nounset diff --git a/hack/deploy-karmada-agent.sh b/hack/deploy-karmada-agent.sh index 8985be36c..485e5df26 100755 --- a/hack/deploy-karmada-agent.sh +++ b/hack/deploy-karmada-agent.sh @@ -85,9 +85,9 @@ kubectl create secret generic karmada-kubeconfig --from-file=karmada-kubeconfig= # deploy karmada agent TEMP_PATH=$(mktemp -d) cp "${REPO_ROOT}"/artifacts/agent/karmada-agent.yaml "${TEMP_PATH}"/karmada-agent.yaml -sed -i "s/{{karmada_context}}/${KARMADA_APISERVER_CONTEXT_NAME}/g" "${TEMP_PATH}"/karmada-agent.yaml -sed -i "s/{{member_cluster_name}}/${MEMBER_CLUSTER_NAME}/g" "${TEMP_PATH}"/karmada-agent.yaml -sed -i "s/{{image_pull_policy}}/${AGENT_IMAGE_PULL_POLICY}/g" "${TEMP_PATH}"/karmada-agent.yaml +sed -i'' -e "s/{{karmada_context}}/${KARMADA_APISERVER_CONTEXT_NAME}/g" "${TEMP_PATH}"/karmada-agent.yaml +sed -i'' -e "s/{{member_cluster_name}}/${MEMBER_CLUSTER_NAME}/g" "${TEMP_PATH}"/karmada-agent.yaml +sed -i'' -e "s/{{image_pull_policy}}/${AGENT_IMAGE_PULL_POLICY}/g" "${TEMP_PATH}"/karmada-agent.yaml echo -e "Apply dynamic rendered deployment in ${TEMP_PATH}/karmada-agent.yaml.\n" kubectl apply -f "${TEMP_PATH}"/karmada-agent.yaml diff --git a/hack/deploy-karmada.sh b/hack/deploy-karmada.sh index 0d5159ee7..82d449c0c 100755 --- a/hack/deploy-karmada.sh +++ b/hack/deploy-karmada.sh @@ -9,7 +9,7 @@ set -o nounset REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. CERT_DIR=${CERT_DIR:-"${HOME}/.karmada"} mkdir -p "${CERT_DIR}" &>/dev/null || sudo mkdir -p "${CERT_DIR}" -rm -f "${CERT_DIR:-${HOME}/.karmada}/*" +rm -f "${CERT_DIR}/*" &>/dev/null || sudo rm -f "${CERT_DIR}/*" KARMADA_APISERVER_SECURE_PORT=${KARMADA_APISERVER_SECURE_PORT:-5443} # The host cluster name which used to install karmada control plane components. @@ -21,10 +21,24 @@ CLUSTER_IP_ONLY=${CLUSTER_IP_ONLY:-false} # whether create a 'ClusterIP' type se source "${REPO_ROOT}"/hack/util.sh function usage() { - echo "This script will deploy karmada control plane to a given cluster." - echo "Usage: hack/deploy-karmada.sh [KARMADA_API_SERVER_IP]" - echo "Example: hack/deploy-karmada.sh ~/.kube/config karmada-host" - unset KUBECONFIG + echo "This script deploys karmada control plane components to a given cluster." + echo "Note: This script is an internal script and is not intended used by end-users." + echo "Usage: hack/deploy-karmada.sh [HOST_CLUSTER_TYPE]" + echo "Example: hack/deploy-karmada.sh ~/.kube/config karmada-host local" + echo -e "Parameters:\n\tKUBECONFIG\t\tYour cluster's kubeconfig that you want to install to" + echo -e "\tCONTEXT_NAME\t\tThe name of context in 'kubeconfig'" + echo -e "\tHOST_CLUSTER_TYPE\tThe type of your cluster that will install Karmada. Optional values are 'local' and 'remote'," + echo -e "\t\t\t\t'local' is default, as that is for the local environment, i.e. for the cluster created by kind." + echo -e "\t\t\t\tAnd if you want to install karmada to a standalone cluster, set it as 'remote'" +} + +# recover the former value of KUBECONFIG +function recover_kubeconfig { + if [ -n "${CURR_KUBECONFIG+x}" ];then + export KUBECONFIG="${CURR_KUBECONFIG}" + else + unset KUBECONFIG + fi } if [[ $# -lt 2 ]]; then @@ -40,17 +54,22 @@ if [[ ! -f "${HOST_CLUSTER_KUBECONFIG}" ]]; then exit 1 fi -# check context existence +# check context existence and switch +# backup current kubeconfig before changing KUBECONFIG +if [ -n "${KUBECONFIG+x}" ];then + CURR_KUBECONFIG=$KUBECONFIG +fi export KUBECONFIG="${HOST_CLUSTER_KUBECONFIG}" HOST_CLUSTER_NAME=$2 -if ! kubectl config get-contexts "${HOST_CLUSTER_NAME}" > /dev/null 2>&1; +if ! kubectl config use-context "${HOST_CLUSTER_NAME}" > /dev/null 2>&1; then - echo -e "ERROR: failed to get context: '${HOST_CLUSTER_NAME}' not in ${HOST_CLUSTER_KUBECONFIG}. \n" + echo -e "ERROR: failed to use context: '${HOST_CLUSTER_NAME}' not in ${HOST_CLUSTER_KUBECONFIG}. \n" usage + recover_kubeconfig exit 1 fi -KARMADA_APISERVER_IP=${3:-} +HOST_CLUSTER_TYPE=${3:-"local"} # the default of host cluster type is local, i.e. cluster created by kind. # generate a secret to store the certificates function generate_cert_secret { @@ -64,16 +83,16 @@ function generate_cert_secret { cp -rf "${REPO_ROOT}"/artifacts/deploy/secret.yaml "${TEMP_PATH}"/secret-tmp.yaml cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-webhook-cert-secret.yaml "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml - sed -i "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml + sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml + sed -i'' -e "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml + sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml - sed -i "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/secret-tmp.yaml - sed -i "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/secret-tmp.yaml - sed -i "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/secret-tmp.yaml + sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/secret-tmp.yaml + sed -i'' -e "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/secret-tmp.yaml + sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/secret-tmp.yaml - sed -i "s/{{server_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml - sed -i "s/{{server_certificate}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml + sed -i'' -e "s/{{server_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml + sed -i'' -e "s/{{server_certificate}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml kubectl apply -f "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml kubectl apply -f "${TEMP_PATH}"/secret-tmp.yaml @@ -81,8 +100,8 @@ function generate_cert_secret { rm -rf "${TEMP_PATH}" } +# install Karmada's APIs function installCRDs() { - # install APIs kubectl apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/cluster.karmada.io_clusters.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/policy.karmada.io_propagationpolicies.yaml" @@ -97,7 +116,7 @@ function installCRDs() { kubectl apply -f "${REPO_ROOT}/artifacts/deploy/multicluster.x-k8s.io_serviceimports.yaml" } -#generate cert +# generate cert util::cmd_must_exist "openssl" util::cmd_must_exist_cfssl ${CFSSL_VERSION} # create CA signers @@ -113,8 +132,8 @@ kubectl apply -f "${REPO_ROOT}/artifacts/deploy/serviceaccount.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/clusterrole.yaml" kubectl apply -f "${REPO_ROOT}/artifacts/deploy/clusterrolebinding.yaml" -KARMADA_CRT=$(sudo base64 "${CERT_DIR}/karmada.crt" | tr -d '\r\n') -KARMADA_KEY=$(sudo base64 "${CERT_DIR}/karmada.key" | tr -d '\r\n') +KARMADA_CRT=$(base64 "${CERT_DIR}/karmada.crt" | tr -d '\r\n') +KARMADA_KEY=$(base64 "${CERT_DIR}/karmada.key" | tr -d '\r\n') generate_cert_secret # deploy karmada etcd @@ -123,37 +142,55 @@ kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-etcd.yaml" # Wait for karmada-etcd to come up before launching the rest of the components. util::wait_pod_ready "${ETCD_POD_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}" -# If it provided a karmada API Server IP we can access karmada API Server (cluster by kind), we will create a ClusterIP type Service -# Or we need to create a LoadBalancer service($CLUSTER_IP_ONLY=false) so that we can access karmada API Server outside the karmada-host cluster +#KARMADA_APISERVER_SERVICE_TYPE is the service type of karmada API Server, For connectivity, it will be different when +# HOST_CLUSTER_TYPE is different. When HOST_CLUSTER_TYPE=local, we will create a ClusterIP type Service. And when +# HOST_CLUSTER_TYPE=remote, we need to create a LoadBalancer service to access Karmada API Server outside the +# karmada-host cluster. Of course, you can still use a ClusterIP type Service by setting $CLUSTER_IP_ONLY=true KARMADA_APISERVER_SERVICE_TYPE="ClusterIP" -if [[ -z "${KARMADA_APISERVER_IP}" ]] && [ "${CLUSTER_IP_ONLY}" = false ]; then - KARMADA_APISERVER_SERVICE_TYPE="LoadBalancer" + +if [ "${HOST_CLUSTER_TYPE}" = "local" ]; then # local mode + KARMADA_APISERVER_IP=$(util::get_apiserver_ip_from_kubeconfig "${HOST_CLUSTER_NAME}") +else # remote mode +# KARMADA_APISERVER_IP will be got when Karmada API Server is ready + if [ "${CLUSTER_IP_ONLY}" = false ]; then + KARMADA_APISERVER_SERVICE_TYPE="LoadBalancer" + fi + HOST_CLUSTER_TYPE="remote" # make sure HOST_CLUSTER_TYPE is in local and remote fi # deploy karmada apiserver TEMP_PATH_APISERVER=$(mktemp -d) cp "${REPO_ROOT}"/artifacts/deploy/karmada-apiserver.yaml "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml -sed -i "s/{{service_type}}/${KARMADA_APISERVER_SERVICE_TYPE}/g" "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml +sed -i'' -e "s/{{service_type}}/${KARMADA_APISERVER_SERVICE_TYPE}/g" "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml echo -e "\nApply dynamic rendered apiserver service in ${TEMP_PATH_APISERVER}/karmada-apiserver.yaml." kubectl apply -f "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml # Wait for karmada-apiserver to come up before launching the rest of the components. util::wait_pod_ready "${APISERVER_POD_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}" -# get Karmada apiserver IP -if [[ -z "${KARMADA_APISERVER_IP}" ]]; then +# get Karmada apiserver IP at remote mode +if [ "${HOST_CLUSTER_TYPE}" = "remote" ]; then case $KARMADA_APISERVER_SERVICE_TYPE in - ClusterIP) KARMADA_APISERVER_IP=$(kubectl get service karmada-apiserver -n "${KARMADA_SYSTEM_NAMESPACE}" -o=jsonpath='{.spec.clusterIP}') + ClusterIP) + KARMADA_APISERVER_IP=$(kubectl get service karmada-apiserver -n "${KARMADA_SYSTEM_NAMESPACE}" -o=jsonpath='{.spec.clusterIP}') ;; - LoadBalancer) if util::wait_service_external_ip "karmada-apiserver" "${KARMADA_SYSTEM_NAMESPACE}"; then - KARMADA_APISERVER_IP=$(util::get_load_balancer_ip) + LoadBalancer) + if util::wait_service_external_ip "karmada-apiserver" "${KARMADA_SYSTEM_NAMESPACE}"; then + echo "Get service external IP: ${SERVICE_EXTERNAL_IP}, wait to check network connectivity" + KARMADA_APISERVER_IP=$(util::get_load_balancer_ip) || KARMADA_APISERVER_IP='' + else + echo "ERROR: wait service external IP timeout, please check the load balancer IP of service: karmada-apiserver" + exit 1 fi ;; esac fi -if [[ -z "${KARMADA_APISERVER_IP}" ]]; then - echo -e "ERROR: failed to get Karmada API server IP after creating service 'karmada-apiserver', please verify.\n" +if [[ -n "${KARMADA_APISERVER_IP}" ]]; then + echo -e "\nKarmada API Server's IP is: ${KARMADA_APISERVER_IP}, host cluster type is: ${HOST_CLUSTER_TYPE}" +else + echo -e "\nERROR: failed to get Karmada API server IP after creating service 'karmada-apiserver' (host cluster type: ${HOST_CLUSTER_TYPE}), please verify." + recover_kubeconfig exit 1 fi @@ -164,12 +201,12 @@ util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/karmada kubectl apply -f "${REPO_ROOT}/artifacts/deploy/kube-controller-manager.yaml" # install CRD APIs on karmada apiserver. -if ! kubectl config get-contexts karmada-apiserver > /dev/null 2>&1; +if ! kubectl config use-context karmada-apiserver > /dev/null 2>&1; then - echo -e "ERROR: failed to get context: karmada-apiserver not in ${HOST_CLUSTER_KUBECONFIG}." + echo -e "ERROR: failed to use context: karmada-apiserver not in ${HOST_CLUSTER_KUBECONFIG}." + recover_kubeconfig exit 1 fi -kubectl config use-context karmada-apiserver installCRDs # deploy webhook configurations on karmada apiserver diff --git a/hack/karmada-bootstrap.sh b/hack/karmada-bootstrap.sh index 102e5320b..fb78dca7a 100755 --- a/hack/karmada-bootstrap.sh +++ b/hack/karmada-bootstrap.sh @@ -40,7 +40,7 @@ util::create_cluster "${PULL_MODE_CLUSTER_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" #step2. make images and get karmadactl export VERSION="latest" export REGISTRY="swr.ap-southeast-1.myhuaweicloud.com/karmada" -make images --directory="${REPO_ROOT}" +make images GOOS="linux" --directory="${REPO_ROOT}" GO111MODULE=on go install "github.com/karmada-io/karmada/cmd/karmadactl" GOPATH=$(go env GOPATH | awk -F ':' '{print $1}') @@ -58,8 +58,7 @@ kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}" #step5. install karmada control plane components -KARMADA_APISERVER_IP=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${HOST_CLUSTER_NAME}-control-plane") -"${REPO_ROOT}"/hack/deploy-karmada.sh "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" "${KARMADA_APISERVER_IP}" +"${REPO_ROOT}"/hack/deploy-karmada.sh "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" #step6. wait until the member cluster ready and join member clusters util::check_clusters_ready "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_1_NAME}" @@ -85,12 +84,9 @@ kind load docker-image "${REGISTRY}/karmada-agent:${VERSION}" --name="${PULL_MOD "${REPO_ROOT}"/hack/deploy-karmada-agent.sh "${MAIN_KUBECONFIG}" "${KARMADA_APISERVER_CLUSTER_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${PULL_MODE_CLUSTER_NAME}" function print_success() { - echo - echo "Local Karmada is running." + echo -e "\nLocal Karmada is running." echo "To start using your karmada, run:" -cat < ./kubectl chmod +x ./kubectl sudo rm -rf "$(which kubectl)" sudo mv ./kubectl /usr/local/bin/kubectl } +function util::install_kind { + local kind_version=${1} + echo "Installing 'kind ${kind_version}' for you, may require your root privileges" + local os_name + os_name=$(go env GOOS) + curl --retry 5 -sSLo ./kind "https://kind.sigs.k8s.io/dl/${kind_version}/kind-${os_name:-linux}-amd64" + chmod +x ./kind + sudo rm -f "$(which kind)" + sudo mv ./kind /usr/local/bin/kind +} + # util::create_signing_certkey creates a CA, args are sudo, dest-dir, ca-id, purpose function util::create_signing_certkey { local sudo=$1 @@ -275,6 +286,27 @@ function util::create_cluster() { echo "Creating cluster ${cluster_name}" } +# This function returns the IP address of a docker instance +# Parameters: +# - $1: docker instance name + +function util::get_docker_native_ipaddress(){ + local container_name=$1 + docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${container_name}" +} + +# This function returns the IP address and port of a specific docker instance's host IP +# Parameters: +# - $1: docker instance name +# Note: +# Use for getting host IP and port for cluster +# "6443/tcp" assumes that API server port is 6443 and protocol is TCP + +function util::get_docker_host_ip_port(){ + local container_name=$1 + docker inspect --format='{{range $key, $value := index .NetworkSettings.Ports "6443/tcp"}}{{if eq $key 0}}{{$value.HostIp}}:{{$value.HostPort}}{{end}}{{end}}' "${container_name}" +} + # util::check_clusters_ready checks if a cluster is ready, if not, wait until timeout function util::check_clusters_ready() { local kubeconfig_path=${1} @@ -285,12 +317,33 @@ function util::check_clusters_ready() { util::wait_for_condition 'running' "docker inspect --format='{{.State.Status}}' ${context_name}-control-plane &> /dev/null" 300 kubectl config rename-context "kind-${context_name}" "${context_name}" --kubeconfig="${kubeconfig_path}" - container_ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${context_name}-control-plane") - kubectl config set-cluster "kind-${context_name}" --server="https://${container_ip}:6443" --kubeconfig="${kubeconfig_path}" + + local os_name + os_name=$(go env GOOS) + local container_ip_port + case $os_name in + linux) container_ip_port=$(util::get_docker_native_ipaddress "${context_name}-control-plane")":6443" + ;; + darwin) container_ip_port=$(util::get_docker_host_ip_port "${context_name}-control-plane") + ;; + *) + echo "OS ${os_name} does NOT support for getting container ip in installation script" + exit 1 + esac + kubectl config set-cluster "kind-${context_name}" --server="https://${container_ip_port}" --kubeconfig="${kubeconfig_path}" util::wait_for_condition 'ok' "kubectl --kubeconfig ${kubeconfig_path} --context ${context_name} get --raw=/healthz &> /dev/null" 300 } +# This function gets api server's ip from kubeconfig by context name +function util::get_apiserver_ip_from_kubeconfig(){ + local context_name=$1 + local cluster_name apiserver_url + cluster_name=$(kubectl config view --template='{{ range $_, $value := .contexts }}{{if eq $value.name '"\"${context_name}\""'}}{{$value.context.cluster}}{{end}}{{end}}') + apiserver_url=$(kubectl config view --template='{{range $_, $value := .clusters }}{{if eq $value.name '"\"${cluster_name}\""'}}{{$value.cluster.server}}{{end}}{{end}}') + echo "${apiserver_url}" | awk -F/ '{print $3}' | sed 's/:.*//' +} + # This function deploys webhook configuration # Parameters: # - $1: CA file @@ -305,7 +358,7 @@ function util::deploy_webhook_configuration() { local temp_path=$(mktemp -d) cp -rf "${conf}" "${temp_path}/temp.yaml" - sed -i "s/{{caBundle}}/${ca_string}/g" "${temp_path}/temp.yaml" + sed -i'' -e "s/{{caBundle}}/${ca_string}/g" "${temp_path}/temp.yaml" kubectl apply -f "${temp_path}/temp.yaml" rm -rf "${temp_path}" }