#!/usr/bin/env bash set -o errexit set -o nounset set -o pipefail # This script holds common bash variables and utility functions. ETCD_POD_LABEL="etcd" APISERVER_POD_LABEL="karmada-apiserver" KUBE_CONTROLLER_POD_LABEL="kube-controller-manager" KARMADA_CONTROLLER_LABEL="karmada-controller-manager" KARMADA_SCHEDULER_LABEL="karmada-scheduler" KARMADA_WEBHOOK_LABEL="karmada-webhook" # This function installs a Go tools by 'go get' command. # Parameters: # - $1: package name, such as "sigs.k8s.io/controller-tools/cmd/controller-gen" # - $2: package version, such as "v0.4.1" # Note: # Since 'go get' command will resolve and add dependencies to current module, that may update 'go.mod' and 'go.sum' file. # So we use a temporary directory to install the tools. function util::install_tools() { local package="$1" local version="$2" temp_path=$(mktemp -d) pushd "${temp_path}" >/dev/null GO111MODULE=on go get "${package}"@"${version}" popd >/dev/null rm -rf "${temp_path}" } # util::cmd_must_exist check whether command is installed. function util::cmd_must_exist { local CMD=$(command -v ${1}) if [[ ! -x ${CMD} ]]; then echo "Please install ${1} and verify they are in \$PATH." exit 1 fi } # util::cmd_must_exist_cfssl downloads cfssl/cfssljson if they do not already exist in PATH function util::cmd_must_exist_cfssl { CFSSL_VERSION=${1} if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null; then CFSSL_BIN=$(command -v cfssl) CFSSLJSON_BIN=$(command -v cfssljson) return 0 fi util::install_tools "github.com/cloudflare/cfssl/cmd/..." ${CFSSL_VERSION} GOPATH=$(go env | grep GOPATH | awk -F '=' '{print $2}'| sed 's/\"//g') CFSSL_BIN="${GOPATH}/bin/cfssl" CFSSLJSON_BIN="${GOPATH}/bin/cfssljson" if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} ]]; then echo "Failed to download 'cfssl'. Please install cfssl and cfssljson and verify they are in \$PATH." echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..." exit 1 fi } # util::create_signing_certkey creates a CA, args are sudo, dest-dir, ca-id, purpose function util::create_signing_certkey { local sudo=$1 local dest_dir=$2 local id=$3 local purpose=$4 OPENSSL_BIN=$(command -v openssl) # Create ca ${sudo} /usr/bin/env bash -e < "${dest_dir}/${id}-ca-config.json" EOF } # util::create_certkey signs a certificate: args are sudo, dest-dir, ca, filename (roughly), subject, hosts... function util::create_certkey { local sudo=$1 local dest_dir=$2 local ca=$3 local id=$4 local cn=${5:-$4} local hosts="" local SEP="" shift 5 while [[ -n "${1:-}" ]]; do hosts+="${SEP}\"$1\"" SEP="," shift 1 done ${sudo} /usr/bin/env bash -e < /dev/null apiVersion: v1 kind: Config clusters: - cluster: "insecure-skip-tls-verify": true server: https://${api_host}:${api_port}/ name: karmada-apiserver users: - user: token: ${token} client-certificate: ${dest_dir}/karmada.crt client-key: ${dest_dir}/karmada.key name: karmada-apiserver contexts: - context: cluster: karmada-apiserver user: karmada-apiserver name: karmada-apiserver current-context: karmada-apiserver EOF ${sudo} chmod 0644 "${dest_dir}"/"${client_id}".config } # util::wait_for_condition blocks until the provided condition becomes true # Arguments: # - 1: message indicating what conditions is being waited for (e.g. 'ok') # - 2: a string representing an eval'able condition. When eval'd it should not output # anything to stdout or stderr. # - 3: optional timeout in seconds. If not provided, waits forever. # Returns: # 1 if the condition is not met before the timeout function util::wait_for_condition() { local msg=$1 # condition should be a string that can be eval'd. local condition=$2 local timeout=${3:-} local start_msg="Waiting for ${msg}" local error_msg="[ERROR] Timeout waiting for ${msg}" local counter=0 while ! eval ${condition}; do if [[ "${counter}" = "0" ]]; then echo -n "${start_msg}" fi if [[ -z "${timeout}" || "${counter}" -lt "${timeout}" ]]; then counter=$((counter + 1)) if [[ -n "${timeout}" ]]; then echo -n '.' fi sleep 1 else echo -e "\n${error_msg}" return 1 fi done if [[ "${counter}" != "0" && -n "${timeout}" ]]; then echo ' done' fi } # util::wait_file_exist checks if a file exists, if not, wait until timeout function util::wait_file_exist() { local file_path=${1} local timeout=${2} for ((time=0; time<${timeout}; time++)); do if [[ -e ${file_path} ]]; then return 0 fi sleep 1 done return 1 } # util::wait_pod_ready waits for pod state becomes ready until timeout. # Parmeters: # - $1: pod label, such as "app=etcd" # - $2: pod namespace, such as "karmada-system" # - $3: time out, such as "200s" function util::wait_pod_ready() { local pod_label=$1 local pod_namespace=$2 echo "wait the $pod_label ready..." set +e util::kubectl_with_retry wait --for=condition=Ready --timeout=200s pods -l app=${pod_label} -n ${pod_namespace} ret=$? set -e return ${ret} } # util::kubectl_with_retry will retry if execute kubectl command failed # tolerate kubectl command failure that may happen before the pod is created by StatefulSet/Deployment. function util::kubectl_with_retry() { local ret=0 for i in `seq 1 30`; do kubectl "$@" ret=$? if [[ ${ret} -ne 0 ]]; then echo "kubectl $@ failed, retrying" sleep 1 continue else return 0 fi done echo "kubectl $@ failed" kubectl "$@" return ${ret} } # util::create_cluster creates a kubernetes cluster function util::create_cluster() { local cluster_name=${1} local kubeconfig=${2} local kind_image=${3} local log_file=${4} rm -rf ${log_file}/${cluster_name}.log mkdir -p ${log_file} nohup kind create cluster --name "${cluster_name}" --kubeconfig="${kubeconfig}" --image="${kind_image}" > ${log_file}/${cluster_name}.log 2>&1 & echo "create cluster ${cluster_name}" } # util::check_clusters_ready checks if a cluster is ready, if not, wait until timeout function util::check_clusters_ready() { local kubeconfig_path=${1} local context_name=${2} echo "Waiting for kubeconfig file ${kubeconfig_path} and clusters ${context_name} to be ready..." util::wait_file_exist ${kubeconfig_path} 120 kubectl config rename-context "kind-${context_name}" "${context_name}" --kubeconfig="${kubeconfig_path}" container_ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${context_name}-control-plane") kubectl config set-cluster "kind-${context_name}" --server="https://${container_ip}:6443" --kubeconfig="${kubeconfig_path}" util::wait_for_condition 'ok' "kubectl --kubeconfig ${kubeconfig_path} --context ${context_name} get --raw=/healthz &> /dev/null" 120 } # This function deploys webhook configuration # Parameters: # - $1: CA file # - $2: configuration file # Note: # Deprecated: should be removed after helm get on board. function util::deploy_webhook_configuration() { local ca_file=$1 local conf=$2 local ca_string=$(sudo cat ${ca_file} | base64 | tr "\n" " "|sed s/[[:space:]]//g) local temp_path=$(mktemp -d) cp -rf "${conf}" "${temp_path}/temp.yaml" sed -i "s/{{caBundle}}/${ca_string}/g" "${temp_path}/temp.yaml" kubectl apply -f "${temp_path}/temp.yaml" rm -rf "${temp_path}" }