Merge pull request #458 from lfbear/issues-434

Karmada-agent install script support the standalone cluster and some misc

README.md

@@ -160,7 +160,7 @@ There are two contexts you can switch after the script run are:
 The `karmada-apiserver` is the **main kubeconfig** to be used when interacting with karamda control plane, while `karmada-host` is only used for debugging karmada installation with host cluster, you can check all clusters at any time by run: `kubectl config view` and switch by `kubectl config use-context [CONTEXT_NAME]`
 
 ##### 3.2. I have present cluster for installing
-Before run the following script, please make sure you are in the node or master of the cluster to install:
+Before running the following script, please make sure your cluster could provide `LoadBalancer` type service.
 ```
 # hack/remote-up-karmada.sh <kubeconfig> <context_name>
 ```
@@ -192,9 +192,15 @@ in `$HOME/.kube/karmada.config`. Run following command:
 ```
 # hack/create-cluster.sh member1 $HOME/.kube/karmada.config
 ```
-The script `hack/create-cluster.sh` will create a standalone cluster by kind.
+The script `hack/create-cluster.sh` will create a cluster by kind.
 
 #### 2. Join member cluster to karmada control plane
+
+You can choose one of mode: [push](#21-push-mode-karmada-controls-the-member-cluster-initiative-by-using-karmadactl) or
+[pull](#22-pull-mode-installing-karmada-agent-in-the-member-cluster), either will help you join a member cluster.
+
+##### 2.1. Push Mode: Karmada controls the member cluster initiative by using `karmadactl`
+
 The command `karmadactl` will help to join the member cluster to karmada control plane,
 before that, we should switch to karmada apiserver:
 ```
@@ -208,7 +214,14 @@ Then, install `karmadactl` command and join the member cluster:
 ```
 The `karmadactl join` command will create a `Cluster` object to reflect the member cluster.
 
-### 3. Check member cluster status
+##### 2.2. Pull Mode: Installing karmada-agent in the member cluster
+
+The following script will install the `karamda-agent` to your member cluster, you need to specify the kubeconfig and the cluster context of the karmada control plane and member cluster.
+```
+# hack/deploy-karmada-agent.sh <karmada_apiserver_kubeconfig> <karmada_apiserver_context_name> <member_cluster_kubeconfig> <member_cluster_context_name>
+```
+
+#### 3. Check member cluster status
 Now, check the member clusters from karmada control plane by following command:
 ```
 # kubectl get clusters

artifacts/agent/karmada-agent.yaml

@@ -22,7 +22,7 @@ spec:
       containers:
         - name: karmada-agent
           image: swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-agent:latest
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: {{image_pull_policy}}
           command:
             - /bin/karmada-agent
             - --karmada-kubeconfig=/etc/kubeconfig/karmada-kubeconfig

artifacts/deploy/karmada-apiserver.yaml

@@ -126,3 +126,4 @@ spec:
       targetPort: 5443
   selector:
     app: karmada-apiserver
+  type: {{service_type}}

hack/deploy-karmada-agent.sh

@@ -57,12 +57,16 @@ CURR_KUBECONFIG=$KUBECONFIG # backup current kubeconfig
 export KUBECONFIG="${MEMBER_CLUSTER_KUBECONFIG}" # switch to member cluster
 kubectl config use-context "${MEMBER_CLUSTER_NAME}"
 
+AGENT_IMAGE_PULL_POLICY="Always" # default is 'always' for standalone member cluster to try to use the latest version
 # load image if member cluster created by kind (if kubeconfig clusters' name contains kind-xxx)
 if kubectl config get-clusters | grep "kind-${MEMBER_CLUSTER_NAME}"
 then
-  export REGISTRY="swr.ap-southeast-1.myhuaweicloud.com/karmada"
+  # make agent image
   export VERSION="latest"
+  export REGISTRY="swr.ap-southeast-1.myhuaweicloud.com/karmada"
+  make image-karmada-agent --directory="${REPO_ROOT}"
   kind load docker-image "${REGISTRY}/karmada-agent:${VERSION}" --name="${MEMBER_CLUSTER_NAME}"
+  AGENT_IMAGE_PULL_POLICY="IfNotPresent" # It must not reload the image when member cluster created by kind
 fi
 
 # create namespace for karmada agent
@@ -81,6 +85,7 @@ TEMP_PATH=$(mktemp -d)
 cp "${REPO_ROOT}"/artifacts/agent/karmada-agent.yaml "${TEMP_PATH}"/karmada-agent.yaml
 sed -i "s/{{karmada_context}}/${KARMADA_APISERVER_CONTEXT_NAME}/g" "${TEMP_PATH}"/karmada-agent.yaml
 sed -i "s/{{member_cluster_name}}/${MEMBER_CLUSTER_NAME}/g" "${TEMP_PATH}"/karmada-agent.yaml
+sed -i "s/{{image_pull_policy}}/${AGENT_IMAGE_PULL_POLICY}/g" "${TEMP_PATH}"/karmada-agent.yaml
 echo -e "Apply dynamic rendered deployment in ${TEMP_PATH}/karmada-agent.yaml.\n"
 kubectl apply -f "${TEMP_PATH}"/karmada-agent.yaml
 

hack/deploy-karmada.sh

@@ -9,6 +9,7 @@ set -o nounset
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 CERT_DIR=${CERT_DIR:-"${HOME}/.karmada"}
 mkdir -p "${CERT_DIR}" &>/dev/null || sudo mkdir -p "${CERT_DIR}"
+rm -f "${CERT_DIR:-${HOME}/.karmada}/*"
 KARMADA_APISERVER_SECURE_PORT=${KARMADA_APISERVER_SECURE_PORT:-5443}
 
 # The host cluster name which used to install karmada control plane components.
@@ -16,7 +17,7 @@ HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"}
 ROOT_CA_FILE=${CERT_DIR}/server-ca.crt
 CFSSL_VERSION="v1.5.0"
 CONTROLPLANE_SUDO=$(test -w "${CERT_DIR}" || echo "sudo -E")
-
+CLUSTER_IP_ONLY=${CLUSTER_IP_ONLY:-false} # whether create a 'ClusterIP' type service for karmada apiserver
 source "${REPO_ROOT}"/hack/util.sh
 
 function usage() {
@@ -49,6 +50,8 @@ then
   exit 1
 fi
 
+KARMADA_APISERVER_IP=${3:-}
+
 # generate a secret to store the certificates
 function generate_cert_secret {
     local karmada_ca
@@ -118,20 +121,37 @@ kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-etcd.yaml"
 # Wait for karmada-etcd to come up before launching the rest of the components.
 util::wait_pod_ready "${ETCD_POD_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
 
+# If it provided a karmada API Server IP we can access karmada API Server (cluster by kind), we will create a ClusterIP type Service
+# Or we need to create a LoadBalancer service($CLUSTER_IP_ONLY=false) so that we can access karmada API Server outside the karmada-host cluster
+KARMADA_APISERVER_SERVICE_TYPE="ClusterIP"
+if [[ -z "${KARMADA_APISERVER_IP}" ]] && [ "${CLUSTER_IP_ONLY}" = false ]; then
+  KARMADA_APISERVER_SERVICE_TYPE="LoadBalancer"
+fi
+
 # deploy karmada apiserver
-kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-apiserver.yaml"
+TEMP_PATH_APISERVER=$(mktemp -d)
+cp "${REPO_ROOT}"/artifacts/deploy/karmada-apiserver.yaml "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml
+sed -i "s/{{service_type}}/${KARMADA_APISERVER_SERVICE_TYPE}/g" "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml
+echo -e "\nApply dynamic rendered apiserver service in ${TEMP_PATH_APISERVER}/karmada-apiserver.yaml."
+kubectl apply -f "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml
 
 # Wait for karmada-apiserver to come up before launching the rest of the components.
 util::wait_pod_ready "${APISERVER_POD_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
 
-if [[ -z "${3-}" ]]; then
+# get Karmada apiserver IP
-  KARMADA_APISERVER_IP=$(kubectl get service karmada-apiserver -n karmada-system -o jsonpath='{.spec.clusterIP}')
+if [[ -z "${KARMADA_APISERVER_IP}" ]]; then
-else
+  case $KARMADA_APISERVER_SERVICE_TYPE in
-  KARMADA_APISERVER_IP=$3
+    ClusterIP)  KARMADA_APISERVER_IP=$(kubectl get service karmada-apiserver -n "${KARMADA_SYSTEM_NAMESPACE}" -o=jsonpath='{.spec.clusterIP}')
+    ;;
+    LoadBalancer)  if util::wait_service_external_ip "karmada-apiserver" "${KARMADA_SYSTEM_NAMESPACE}"; then
+      KARMADA_APISERVER_IP=$(util::get_load_balancer_ip)
+      fi
+    ;;
+  esac
 fi
 
 if [[ -z "${KARMADA_APISERVER_IP}" ]]; then
-  echo -e "ERROR: failed to create service 'karmada-apiserver', please verify.\n"
+  echo -e "ERROR: failed to get Karmada API server IP after creating service 'karmada-apiserver', please verify.\n"
   exit 1
 fi
 

hack/remote-up-karmada.sh

@@ -6,11 +6,16 @@ set -o pipefail
 
 function usage() {
   echo "This script will deploy karmada control plane to a given cluster."
-  echo "Usage: hack/remote-up-karmada.sh <KUBECONFIG> <CONTEXT_NAME>"
+  echo "Usage: hack/remote-up-karmada.sh <KUBECONFIG> <CONTEXT_NAME> [CLUSTER_IP_ONLY]"
   echo "Example: hack/remote-up-karmada.sh ~/.kube/config karmada-host"
+  echo -e "Parameters:\n\tKUBECONFIG\tYour cluster's kubeconfig that you want to install to"
+  echo -e "\tCONTEXT_NAME\tThe name of context in 'kubeconfig'"
+  echo -e "\tCLUSTER_IP_ONLY\tThis option default is 'false', and there will create a 'LoadBalancer' type service
+  \t\t\tfor karmada apiserver so that it can easy communicate outside the karmada-host cluster,
+  \t\t\tif you want only a 'ClusterIP' type service for karmada apiserver, set as 'true'."
 }
 
-if [[ $# -ne 2 ]]; then
+if [[ $# -lt 2 ]]; then
   usage
   exit 1
 fi
@@ -33,6 +38,11 @@ then
   exit 1
 fi
 
+if [ "${3:-false}" = true ]; then
+  CLUSTER_IP_ONLY=true
+  export CLUSTER_IP_ONLY
+fi
+
 # deploy karmada control plane
 SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 "${SCRIPT_ROOT}"/hack/deploy-karmada.sh "${HOST_CLUSTER_KUBECONFIG}" "${HOST_CLUSTER_NAME}"

hack/undeploy-karmada.sh

@@ -6,14 +6,37 @@ set -o pipefail
 
 function usage() {
   echo "This script will remove karmada control plane from a cluster."
-  echo "Usage: hack/undeploy-karmada.sh"
+  echo "Usage: hack/undeploy-karmada.sh [KUBECONFIG] [CONTEXT_NAME]"
-  echo "Example: hack/undeploy-karmada.sh"
+  echo "Example: hack/undeploy-karmada.sh ~/.kube/karmada.config karmada-host"
 }
 
 SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-
+# delete all keys and certificates
+rm -fr "${HOME}/.karmada"
+# set default host cluster's kubeconfig file (created by kind)
 KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
-HOST_CLUSTER_KUBECONFIG="${KUBECONFIG_PATH}/karmada-host.config"
+HOST_CLUSTER_KUBECONFIG="${KUBECONFIG_PATH}/karmada.config"
+HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"}
 
-export KUBECONFIG=${HOST_CLUSTER_KUBECONFIG}
+# if provider the custom kubeconfig and context name
-kubectl delete ns karmada-system
+if [[ -f "${1:-}" ]]; then
+  if ! kubectl config get-contexts "${2:-}" --kubeconfig="${1}" > /dev/null 2>&1;
+  then
+    echo -e "ERROR: failed to get context: '${2:-}' not in ${1}. \n"
+    usage
+    exit 1
+  else
+    HOST_CLUSTER_KUBECONFIG=${1:-}
+    HOST_CLUSTER_NAME=${2:-}
+  fi
+fi
+
+kubectl config use-context "${HOST_CLUSTER_NAME}" --kubeconfig="${HOST_CLUSTER_KUBECONFIG}"
+
+# clear all in namespace karmada-system
+kubectl delete ns karmada-system --kubeconfig="${HOST_CLUSTER_KUBECONFIG}"
+
+# clear configs about karmada-apiserver in kubeconfig
+kubectl config delete-cluster karmada-apiserver --kubeconfig="${HOST_CLUSTER_KUBECONFIG}"
+kubectl config delete-user karmada-apiserver --kubeconfig="${HOST_CLUSTER_KUBECONFIG}"
+kubectl config delete-context karmada-apiserver --kubeconfig="${HOST_CLUSTER_KUBECONFIG}"

hack/util.sh

@@ -295,3 +295,54 @@ function util::deploy_webhook_configuration() {
   kubectl apply -f "${temp_path}/temp.yaml"
   rm -rf "${temp_path}"
 }
+
+# util::wait_service_external_ip give a service external ip when it is ready, if not, wait until timeout
+# Parameters:
+#  - $1: service name in k8s
+#  - $2: namespace
+SERVICE_EXTERNAL_IP=''
+function util::wait_service_external_ip() {
+  local service_name=$1
+  local namespace=$2
+  local external_ip
+  local tmp
+  for tmp in {1..30}; do
+    set +e
+    external_ip=$(kubectl get service "${service_name}" -n "${namespace}" --template="{{range .status.loadBalancer.ingress}}{{.ip}} {{end}}" | xargs)
+    set -e
+    if [[ -z "$external_ip" ]]; then
+      echo "wait the external ip of ${service_name} ready..."
+      sleep 6
+      continue
+    else
+      SERVICE_EXTERNAL_IP="${external_ip}"
+      return 0
+    fi
+  done
+  return 1
+}
+
+# util::get_load_balancer_ip get a valid load balancer ip from k8s service's 'loadBalancer' , if not, wait until timeout
+# call 'util::wait_service_external_ip' before using this function
+function util::get_load_balancer_ip() {
+  local tmp
+  local first_ip
+  if [[ -n "${SERVICE_EXTERNAL_IP}" ]]; then
+    first_ip=$(echo "${SERVICE_EXTERNAL_IP}" | awk '{print $1}') #temporarily choose the first one
+    for tmp in {1..10}; do
+      set +e
+      connect_test=$(curl -s -k -m 5 https://"${first_ip}":5443/readyz)
+      set -e
+      if [[ "${connect_test}" = "ok" ]]; then
+        echo "${first_ip}"
+        return 0
+      else
+        sleep 3
+        continue
+      fi
+    done
+  fi
+  return 1
+}
+
+