diff --git a/artifacts/kindClusterConfig/member1.yaml b/artifacts/kindClusterConfig/member1.yaml new file mode 100644 index 000000000..5401c71ce --- /dev/null +++ b/artifacts/kindClusterConfig/member1.yaml @@ -0,0 +1,9 @@ +kind: Cluster +apiVersion: "kind.x-k8s.io/v1alpha4" +networking: + podSubnet: "10.10.0.0/16" + serviceSubnet: "10.11.0.0/16" +featureGates: + EndpointSliceProxying: true +nodes: + - role: control-plane diff --git a/artifacts/kindClusterConfig/member2.yaml b/artifacts/kindClusterConfig/member2.yaml new file mode 100644 index 000000000..aaf3042ce --- /dev/null +++ b/artifacts/kindClusterConfig/member2.yaml @@ -0,0 +1,9 @@ +kind: Cluster +apiVersion: "kind.x-k8s.io/v1alpha4" +networking: + podSubnet: "10.12.0.0/16" + serviceSubnet: "10.13.0.0/16" +featureGates: + EndpointSliceProxying: true +nodes: + - role: control-plane diff --git a/hack/karmada-bootstrap.sh b/hack/karmada-bootstrap.sh index bb8dbe183..102e5320b 100755 --- a/hack/karmada-bootstrap.sh +++ b/hack/karmada-bootstrap.sh @@ -25,13 +25,16 @@ CLUSTER_VERSION=${CLUSTER_VERSION:-"kindest/node:v1.19.1"} KIND_LOG_FILE=${KIND_LOG_FILE:-"/tmp/karmada"} #step0: prepare -# install kind +# install kind and kubectl util::install_tools sigs.k8s.io/kind v0.10.0 +# we choose v1.18.0, because in kubectl after versions 1.18 exist a bug which will give wrong output when using jsonpath. +# bug details: https://github.com/kubernetes/kubernetes/pull/98057 +util::install_kubectl "v1.18.0" "amd64" #step1. create host cluster and member clusters in parallel util::create_cluster "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" -util::create_cluster "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" -util::create_cluster "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" +util::create_cluster "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}/artifacts/kindClusterConfig/member1.yaml" +util::create_cluster "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${REPO_ROOT}/artifacts/kindClusterConfig/member2.yaml" util::create_cluster "${PULL_MODE_CLUSTER_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" #step2. make images and get karmadactl @@ -62,6 +65,12 @@ KARMADA_APISERVER_IP=$(docker inspect --format='{{range .NetworkSettings.Network util::check_clusters_ready "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_1_NAME}" util::check_clusters_ready "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_2_NAME}" +# connecting networks between member1 and member2 clusters +echo "connecting cluster networks..." +util::add_routes "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_2_NAME}" +util::add_routes "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_1_NAME}" +echo "cluster networks connected" + #join push mode member clusters export KUBECONFIG="${MAIN_KUBECONFIG}" kubectl config use-context "${KARMADA_APISERVER_CLUSTER_NAME}" diff --git a/hack/util.sh b/hack/util.sh index f4d1257f8..4eed595fd 100755 --- a/hack/util.sh +++ b/hack/util.sh @@ -65,6 +65,17 @@ function util::cmd_must_exist_cfssl { fi } +# util::install_kubectl will install the given version kubectl +function util::install_kubectl { + local KUBECTL_VERSION=${1} + local ARCH=${2} + + curl -sSL --retry 5 https://dl.k8s.io/release/"$KUBECTL_VERSION"/bin/linux/"$ARCH"/kubectl > ./kubectl + chmod +x ./kubectl + sudo rm -rf "$(which kubectl)" + sudo mv ./kubectl /usr/local/bin/kubectl +} + # util::create_signing_certkey creates a CA, args are sudo, dest-dir, ca-id, purpose function util::create_signing_certkey { local sudo=$1 @@ -255,11 +266,12 @@ function util::create_cluster() { local kubeconfig=${2} local kind_image=${3} local log_path=${4} + local cluster_config=${5:-} mkdir -p ${log_path} rm -rf "${log_path}/${cluster_name}.log" rm -f "${kubeconfig}" - nohup kind delete cluster --name="${cluster_name}" >> "${log_path}"/"${cluster_name}".log 2>&1 && kind create cluster --name "${cluster_name}" --kubeconfig="${kubeconfig}" --image="${kind_image}" >> "${log_path}"/"${cluster_name}".log 2>&1 & + nohup kind delete cluster --name="${cluster_name}" >> "${log_path}"/"${cluster_name}".log 2>&1 && kind create cluster --name "${cluster_name}" --kubeconfig="${kubeconfig}" --image="${kind_image}" --config="${cluster_config}" >> "${log_path}"/"${cluster_name}".log 2>&1 & echo "Creating cluster ${cluster_name}" } @@ -345,4 +357,23 @@ function util::get_load_balancer_ip() { return 1 } +# util::add_routes will add routes for given kind cluster +# Parameters: +# - $1: name of the kind cluster want to add routes +# - $2: the kubeconfig path of the cluster wanted to be connected +# - $3: the context in kubeconfig of the cluster wanted to be connected +function util::add_routes() { + unset IFS + routes=$(kubectl --kubeconfig ${2} --context ${3} get nodes -o jsonpath='{range .items[*]}ip route add {.spec.podCIDR} via {.status.addresses[?(.type=="InternalIP")].address}{"\n"}') + echo "Connecting cluster ${1} to ${2}" + + IFS=$'\n' + for n in $(kind get nodes --name "${1}"); do + for r in $routes; do + echo "exec cmd in docker $n $r" + eval "docker exec $n $r" + done + done + unset IFS +}