diff --git a/README.md b/README.md index aaede2b8d..591e00381 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ There are two contexts you can switch after the script run are: The `karmada-apiserver` is the **main kubeconfig** to be used when interacting with karamda control plane, while `karmada-host` is only used for debugging karmada installation with host cluster, you can check all clusters at any time by run: `kubectl config view` and switch by `kubectl config use-context [CONTEXT_NAME]` ##### 3.2. I have present cluster for installing -Before run the following script, please make sure you are in the node or master of the cluster to install: +Before running the following script, please make sure your cluster could provide `LoadBalancer` type service. ``` # hack/remote-up-karmada.sh ``` @@ -192,9 +192,15 @@ in `$HOME/.kube/karmada.config`. Run following command: ``` # hack/create-cluster.sh member1 $HOME/.kube/karmada.config ``` -The script `hack/create-cluster.sh` will create a standalone cluster by kind. +The script `hack/create-cluster.sh` will create a cluster by kind. #### 2. Join member cluster to karmada control plane + +You can choose one of mode: [push](#21-push-mode-karmada-controls-the-member-cluster-initiative-by-using-karmadactl) or +[pull](#22-pull-mode-installing-karmada-agent-in-the-member-cluster), either will help you join a member cluster. + +##### 2.1. Push Mode: Karmada controls the member cluster initiative by using `karmadactl` + The command `karmadactl` will help to join the member cluster to karmada control plane, before that, we should switch to karmada apiserver: ``` @@ -208,7 +214,14 @@ Then, install `karmadactl` command and join the member cluster: ``` The `karmadactl join` command will create a `Cluster` object to reflect the member cluster. -### 3. Check member cluster status +##### 2.2. Pull Mode: Installing karmada-agent in the member cluster + +The following script will install the `karamda-agent` to your member cluster, you need to specify the kubeconfig and the cluster context of the karmada control plane and member cluster. +``` +# hack/deploy-karmada-agent.sh +``` + +#### 3. Check member cluster status Now, check the member clusters from karmada control plane by following command: ``` # kubectl get clusters diff --git a/artifacts/agent/karmada-agent.yaml b/artifacts/agent/karmada-agent.yaml index 88fb6d50a..4c77b0792 100644 --- a/artifacts/agent/karmada-agent.yaml +++ b/artifacts/agent/karmada-agent.yaml @@ -22,7 +22,7 @@ spec: containers: - name: karmada-agent image: swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-agent:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: {{image_pull_policy}} command: - /bin/karmada-agent - --karmada-kubeconfig=/etc/kubeconfig/karmada-kubeconfig diff --git a/artifacts/deploy/karmada-apiserver.yaml b/artifacts/deploy/karmada-apiserver.yaml index 7eaa48abb..ea964e262 100644 --- a/artifacts/deploy/karmada-apiserver.yaml +++ b/artifacts/deploy/karmada-apiserver.yaml @@ -126,3 +126,4 @@ spec: targetPort: 5443 selector: app: karmada-apiserver + type: {{service_type}} diff --git a/hack/deploy-karmada-agent.sh b/hack/deploy-karmada-agent.sh index 1ccdeea99..9657af9f4 100755 --- a/hack/deploy-karmada-agent.sh +++ b/hack/deploy-karmada-agent.sh @@ -57,12 +57,16 @@ CURR_KUBECONFIG=$KUBECONFIG # backup current kubeconfig export KUBECONFIG="${MEMBER_CLUSTER_KUBECONFIG}" # switch to member cluster kubectl config use-context "${MEMBER_CLUSTER_NAME}" +AGENT_IMAGE_PULL_POLICY="Always" # default is 'always' for standalone member cluster to try to use the latest version # load image if member cluster created by kind (if kubeconfig clusters' name contains kind-xxx) if kubectl config get-clusters | grep "kind-${MEMBER_CLUSTER_NAME}" then - export REGISTRY="swr.ap-southeast-1.myhuaweicloud.com/karmada" + # make agent image export VERSION="latest" + export REGISTRY="swr.ap-southeast-1.myhuaweicloud.com/karmada" + make image-karmada-agent --directory="${REPO_ROOT}" kind load docker-image "${REGISTRY}/karmada-agent:${VERSION}" --name="${MEMBER_CLUSTER_NAME}" + AGENT_IMAGE_PULL_POLICY="IfNotPresent" # It must not reload the image when member cluster created by kind fi # create namespace for karmada agent @@ -81,6 +85,7 @@ TEMP_PATH=$(mktemp -d) cp "${REPO_ROOT}"/artifacts/agent/karmada-agent.yaml "${TEMP_PATH}"/karmada-agent.yaml sed -i "s/{{karmada_context}}/${KARMADA_APISERVER_CONTEXT_NAME}/g" "${TEMP_PATH}"/karmada-agent.yaml sed -i "s/{{member_cluster_name}}/${MEMBER_CLUSTER_NAME}/g" "${TEMP_PATH}"/karmada-agent.yaml +sed -i "s/{{image_pull_policy}}/${AGENT_IMAGE_PULL_POLICY}/g" "${TEMP_PATH}"/karmada-agent.yaml echo -e "Apply dynamic rendered deployment in ${TEMP_PATH}/karmada-agent.yaml.\n" kubectl apply -f "${TEMP_PATH}"/karmada-agent.yaml diff --git a/hack/deploy-karmada.sh b/hack/deploy-karmada.sh index 262bbcece..e233ec6cd 100755 --- a/hack/deploy-karmada.sh +++ b/hack/deploy-karmada.sh @@ -9,6 +9,7 @@ set -o nounset REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. CERT_DIR=${CERT_DIR:-"${HOME}/.karmada"} mkdir -p "${CERT_DIR}" &>/dev/null || sudo mkdir -p "${CERT_DIR}" +rm -f "${CERT_DIR:-${HOME}/.karmada}/*" KARMADA_APISERVER_SECURE_PORT=${KARMADA_APISERVER_SECURE_PORT:-5443} # The host cluster name which used to install karmada control plane components. @@ -16,7 +17,7 @@ HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"} ROOT_CA_FILE=${CERT_DIR}/server-ca.crt CFSSL_VERSION="v1.5.0" CONTROLPLANE_SUDO=$(test -w "${CERT_DIR}" || echo "sudo -E") - +CLUSTER_IP_ONLY=${CLUSTER_IP_ONLY:-false} # whether create a 'ClusterIP' type service for karmada apiserver source "${REPO_ROOT}"/hack/util.sh function usage() { @@ -49,6 +50,8 @@ then exit 1 fi +KARMADA_APISERVER_IP=${3:-} + # generate a secret to store the certificates function generate_cert_secret { local karmada_ca @@ -118,20 +121,37 @@ kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-etcd.yaml" # Wait for karmada-etcd to come up before launching the rest of the components. util::wait_pod_ready "${ETCD_POD_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}" +# If it provided a karmada API Server IP we can access karmada API Server (cluster by kind), we will create a ClusterIP type Service +# Or we need to create a LoadBalancer service($CLUSTER_IP_ONLY=false) so that we can access karmada API Server outside the karmada-host cluster +KARMADA_APISERVER_SERVICE_TYPE="ClusterIP" +if [[ -z "${KARMADA_APISERVER_IP}" ]] && [ "${CLUSTER_IP_ONLY}" = false ]; then + KARMADA_APISERVER_SERVICE_TYPE="LoadBalancer" +fi + # deploy karmada apiserver -kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-apiserver.yaml" +TEMP_PATH_APISERVER=$(mktemp -d) +cp "${REPO_ROOT}"/artifacts/deploy/karmada-apiserver.yaml "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml +sed -i "s/{{service_type}}/${KARMADA_APISERVER_SERVICE_TYPE}/g" "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml +echo -e "\nApply dynamic rendered apiserver service in ${TEMP_PATH_APISERVER}/karmada-apiserver.yaml." +kubectl apply -f "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml # Wait for karmada-apiserver to come up before launching the rest of the components. util::wait_pod_ready "${APISERVER_POD_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}" -if [[ -z "${3-}" ]]; then - KARMADA_APISERVER_IP=$(kubectl get service karmada-apiserver -n karmada-system -o jsonpath='{.spec.clusterIP}') -else - KARMADA_APISERVER_IP=$3 +# get Karmada apiserver IP +if [[ -z "${KARMADA_APISERVER_IP}" ]]; then + case $KARMADA_APISERVER_SERVICE_TYPE in + ClusterIP) KARMADA_APISERVER_IP=$(kubectl get service karmada-apiserver -n "${KARMADA_SYSTEM_NAMESPACE}" -o=jsonpath='{.spec.clusterIP}') + ;; + LoadBalancer) if util::wait_service_external_ip "karmada-apiserver" "${KARMADA_SYSTEM_NAMESPACE}"; then + KARMADA_APISERVER_IP=$(util::get_load_balancer_ip) + fi + ;; + esac fi if [[ -z "${KARMADA_APISERVER_IP}" ]]; then - echo -e "ERROR: failed to create service 'karmada-apiserver', please verify.\n" + echo -e "ERROR: failed to get Karmada API server IP after creating service 'karmada-apiserver', please verify.\n" exit 1 fi diff --git a/hack/remote-up-karmada.sh b/hack/remote-up-karmada.sh index a80754f1a..a807cfeea 100755 --- a/hack/remote-up-karmada.sh +++ b/hack/remote-up-karmada.sh @@ -6,11 +6,16 @@ set -o pipefail function usage() { echo "This script will deploy karmada control plane to a given cluster." - echo "Usage: hack/remote-up-karmada.sh " + echo "Usage: hack/remote-up-karmada.sh [CLUSTER_IP_ONLY]" echo "Example: hack/remote-up-karmada.sh ~/.kube/config karmada-host" + echo -e "Parameters:\n\tKUBECONFIG\tYour cluster's kubeconfig that you want to install to" + echo -e "\tCONTEXT_NAME\tThe name of context in 'kubeconfig'" + echo -e "\tCLUSTER_IP_ONLY\tThis option default is 'false', and there will create a 'LoadBalancer' type service + \t\t\tfor karmada apiserver so that it can easy communicate outside the karmada-host cluster, + \t\t\tif you want only a 'ClusterIP' type service for karmada apiserver, set as 'true'." } -if [[ $# -ne 2 ]]; then +if [[ $# -lt 2 ]]; then usage exit 1 fi @@ -33,6 +38,11 @@ then exit 1 fi +if [ "${3:-false}" = true ]; then + CLUSTER_IP_ONLY=true + export CLUSTER_IP_ONLY +fi + # deploy karmada control plane SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. "${SCRIPT_ROOT}"/hack/deploy-karmada.sh "${HOST_CLUSTER_KUBECONFIG}" "${HOST_CLUSTER_NAME}" diff --git a/hack/undeploy-karmada.sh b/hack/undeploy-karmada.sh index 4c186fa41..d8f0d53ee 100755 --- a/hack/undeploy-karmada.sh +++ b/hack/undeploy-karmada.sh @@ -6,14 +6,37 @@ set -o pipefail function usage() { echo "This script will remove karmada control plane from a cluster." - echo "Usage: hack/undeploy-karmada.sh" - echo "Example: hack/undeploy-karmada.sh" + echo "Usage: hack/undeploy-karmada.sh [KUBECONFIG] [CONTEXT_NAME]" + echo "Example: hack/undeploy-karmada.sh ~/.kube/karmada.config karmada-host" } SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. - +# delete all keys and certificates +rm -fr "${HOME}/.karmada" +# set default host cluster's kubeconfig file (created by kind) KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} -HOST_CLUSTER_KUBECONFIG="${KUBECONFIG_PATH}/karmada-host.config" +HOST_CLUSTER_KUBECONFIG="${KUBECONFIG_PATH}/karmada.config" +HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"} -export KUBECONFIG=${HOST_CLUSTER_KUBECONFIG} -kubectl delete ns karmada-system +# if provider the custom kubeconfig and context name +if [[ -f "${1:-}" ]]; then + if ! kubectl config get-contexts "${2:-}" --kubeconfig="${1}" > /dev/null 2>&1; + then + echo -e "ERROR: failed to get context: '${2:-}' not in ${1}. \n" + usage + exit 1 + else + HOST_CLUSTER_KUBECONFIG=${1:-} + HOST_CLUSTER_NAME=${2:-} + fi +fi + +kubectl config use-context "${HOST_CLUSTER_NAME}" --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" + +# clear all in namespace karmada-system +kubectl delete ns karmada-system --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" + +# clear configs about karmada-apiserver in kubeconfig +kubectl config delete-cluster karmada-apiserver --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" +kubectl config delete-user karmada-apiserver --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" +kubectl config delete-context karmada-apiserver --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" diff --git a/hack/util.sh b/hack/util.sh index 747d31ec4..f4d1257f8 100755 --- a/hack/util.sh +++ b/hack/util.sh @@ -295,3 +295,54 @@ function util::deploy_webhook_configuration() { kubectl apply -f "${temp_path}/temp.yaml" rm -rf "${temp_path}" } + +# util::wait_service_external_ip give a service external ip when it is ready, if not, wait until timeout +# Parameters: +# - $1: service name in k8s +# - $2: namespace +SERVICE_EXTERNAL_IP='' +function util::wait_service_external_ip() { + local service_name=$1 + local namespace=$2 + local external_ip + local tmp + for tmp in {1..30}; do + set +e + external_ip=$(kubectl get service "${service_name}" -n "${namespace}" --template="{{range .status.loadBalancer.ingress}}{{.ip}} {{end}}" | xargs) + set -e + if [[ -z "$external_ip" ]]; then + echo "wait the external ip of ${service_name} ready..." + sleep 6 + continue + else + SERVICE_EXTERNAL_IP="${external_ip}" + return 0 + fi + done + return 1 +} + +# util::get_load_balancer_ip get a valid load balancer ip from k8s service's 'loadBalancer' , if not, wait until timeout +# call 'util::wait_service_external_ip' before using this function +function util::get_load_balancer_ip() { + local tmp + local first_ip + if [[ -n "${SERVICE_EXTERNAL_IP}" ]]; then + first_ip=$(echo "${SERVICE_EXTERNAL_IP}" | awk '{print $1}') #temporarily choose the first one + for tmp in {1..10}; do + set +e + connect_test=$(curl -s -k -m 5 https://"${first_ip}":5443/readyz) + set -e + if [[ "${connect_test}" = "ok" ]]; then + echo "${first_ip}" + return 0 + else + sleep 3 + continue + fi + done + fi + return 1 +} + +