Merge pull request #501 from mrlihanbo/kind-cluster-connect

add cluster connectivity in e2e

artifacts/kindClusterConfig/member1.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: "kind.x-k8s.io/v1alpha4"
+networking:
+  podSubnet: "10.10.0.0/16"
+  serviceSubnet: "10.11.0.0/16"
+featureGates:
+  EndpointSliceProxying: true
+nodes:
+  - role: control-plane

artifacts/kindClusterConfig/member2.yaml

@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: "kind.x-k8s.io/v1alpha4"
+networking:
+  podSubnet: "10.12.0.0/16"
+  serviceSubnet: "10.13.0.0/16"
+featureGates:
+  EndpointSliceProxying: true
+nodes:
+  - role: control-plane

hack/karmada-bootstrap.sh

@@ -25,13 +25,16 @@ CLUSTER_VERSION=${CLUSTER_VERSION:-"kindest/node:v1.19.1"}
 KIND_LOG_FILE=${KIND_LOG_FILE:-"/tmp/karmada"}
 
 #step0: prepare
-# install kind
+# install kind and kubectl
 util::install_tools sigs.k8s.io/kind v0.10.0
+# we choose v1.18.0, because in kubectl after versions 1.18 exist a bug which will give wrong output when using jsonpath.
+# bug details: https://github.com/kubernetes/kubernetes/pull/98057
+util::install_kubectl "v1.18.0" "amd64"
 
 #step1. create host cluster and member clusters in parallel
 util::create_cluster "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}"
-util::create_cluster "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}"
+util::create_cluster "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}/artifacts/kindClusterConfig/member1.yaml"
-util::create_cluster "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}"
+util::create_cluster "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}/artifacts/kindClusterConfig/member2.yaml"
 util::create_cluster "${PULL_MODE_CLUSTER_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}"
 
 #step2. make images and get karmadactl
@@ -62,6 +65,12 @@ KARMADA_APISERVER_IP=$(docker inspect --format='{{range .NetworkSettings.Network
 util::check_clusters_ready "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_1_NAME}"
 util::check_clusters_ready "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_2_NAME}"
 
+# connecting networks between member1 and member2 clusters
+echo "connecting cluster networks..."
+util::add_routes "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_2_NAME}"
+util::add_routes "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_1_NAME}"
+echo "cluster networks connected"
+
 #join push mode member clusters
 export KUBECONFIG="${MAIN_KUBECONFIG}"
 kubectl config use-context "${KARMADA_APISERVER_CLUSTER_NAME}"

hack/util.sh

@@ -65,6 +65,17 @@ function util::cmd_must_exist_cfssl {
     fi
 }
 
+# util::install_kubectl will install the given version kubectl
+function util::install_kubectl {
+    local KUBECTL_VERSION=${1}
+    local ARCH=${2}
+
+    curl -sSL --retry 5 https://dl.k8s.io/release/"$KUBECTL_VERSION"/bin/linux/"$ARCH"/kubectl > ./kubectl
+    chmod +x ./kubectl
+    sudo rm -rf "$(which kubectl)"
+    sudo mv ./kubectl /usr/local/bin/kubectl
+}
+
 # util::create_signing_certkey creates a CA, args are sudo, dest-dir, ca-id, purpose
 function util::create_signing_certkey {
     local sudo=$1
@@ -255,11 +266,12 @@ function util::create_cluster() {
   local kubeconfig=${2}
   local kind_image=${3}
   local log_path=${4}
+  local cluster_config=${5:-}
 
   mkdir -p ${log_path}
   rm -rf "${log_path}/${cluster_name}.log"
   rm -f "${kubeconfig}"
-  nohup kind delete cluster --name="${cluster_name}" >> "${log_path}"/"${cluster_name}".log 2>&1 && kind create cluster --name "${cluster_name}" --kubeconfig="${kubeconfig}" --image="${kind_image}" >> "${log_path}"/"${cluster_name}".log 2>&1 &
+  nohup kind delete cluster --name="${cluster_name}" >> "${log_path}"/"${cluster_name}".log 2>&1 && kind create cluster --name "${cluster_name}" --kubeconfig="${kubeconfig}" --image="${kind_image}" --config="${cluster_config}" >> "${log_path}"/"${cluster_name}".log 2>&1 &
   echo "Creating cluster ${cluster_name}"
 }
 
@@ -345,4 +357,23 @@ function util::get_load_balancer_ip() {
   return 1
 }
 
+# util::add_routes will add routes for given kind cluster
+# Parameters:
+#  - $1: name of the kind cluster want to add routes
+#  - $2: the kubeconfig path of the cluster wanted to be connected
+#  - $3: the context in kubeconfig of the cluster wanted to be connected
+function util::add_routes() {
+  unset IFS
+  routes=$(kubectl --kubeconfig ${2} --context ${3} get nodes -o jsonpath='{range .items[*]}ip route add {.spec.podCIDR} via {.status.addresses[?(.type=="InternalIP")].address}{"\n"}')
+  echo "Connecting cluster ${1} to ${2}"
+
+  IFS=$'\n'
+  for n in $(kind get nodes --name "${1}"); do
+    for r in $routes; do
+      echo "exec cmd in docker $n $r"
+      eval "docker exec $n $r"
+    done
+  done
+  unset IFS
+}