382 lines
13 KiB
Bash
Executable File
382 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -o errexit
|
|
set -o nounset
|
|
set -o pipefail
|
|
|
|
# This script holds common bash variables and utility functions.
|
|
|
|
KARMADA_SYSTEM_NAMESPACE="karmada-system"
|
|
ETCD_POD_LABEL="etcd"
|
|
APISERVER_POD_LABEL="karmada-apiserver"
|
|
KUBE_CONTROLLER_POD_LABEL="kube-controller-manager"
|
|
KARMADA_CONTROLLER_LABEL="karmada-controller-manager"
|
|
KARMADA_SCHEDULER_LABEL="karmada-scheduler"
|
|
KARMADA_WEBHOOK_LABEL="karmada-webhook"
|
|
AGENT_POD_LABEL="karmada-agent"
|
|
|
|
# This function installs a Go tools by 'go get' command.
|
|
# Parameters:
|
|
# - $1: package name, such as "sigs.k8s.io/controller-tools/cmd/controller-gen"
|
|
# - $2: package version, such as "v0.4.1"
|
|
# Note:
|
|
# Since 'go get' command will resolve and add dependencies to current module, that may update 'go.mod' and 'go.sum' file.
|
|
# So we use a temporary directory to install the tools.
|
|
function util::install_tools() {
|
|
local package="$1"
|
|
local version="$2"
|
|
|
|
temp_path=$(mktemp -d)
|
|
pushd "${temp_path}" >/dev/null
|
|
GO111MODULE=on go get "${package}"@"${version}"
|
|
GOPATH=$(go env GOPATH | awk -F ':' '{print $1}')
|
|
export PATH=$PATH:$GOPATH/bin
|
|
popd >/dev/null
|
|
rm -rf "${temp_path}"
|
|
}
|
|
|
|
# util::cmd_must_exist check whether command is installed.
|
|
function util::cmd_must_exist {
|
|
local CMD=$(command -v ${1})
|
|
if [[ ! -x ${CMD} ]]; then
|
|
echo "Please install ${1} and verify they are in \$PATH."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# util::cmd_must_exist_cfssl downloads cfssl/cfssljson if they do not already exist in PATH
|
|
function util::cmd_must_exist_cfssl {
|
|
CFSSL_VERSION=${1}
|
|
if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null; then
|
|
CFSSL_BIN=$(command -v cfssl)
|
|
CFSSLJSON_BIN=$(command -v cfssljson)
|
|
return 0
|
|
fi
|
|
|
|
util::install_tools "github.com/cloudflare/cfssl/cmd/..." ${CFSSL_VERSION}
|
|
|
|
GOPATH=$(go env GOPATH | awk -F ':' '{print $1}')
|
|
CFSSL_BIN="${GOPATH}/bin/cfssl"
|
|
CFSSLJSON_BIN="${GOPATH}/bin/cfssljson"
|
|
if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} ]]; then
|
|
echo "Failed to download 'cfssl'. Please install cfssl and cfssljson and verify they are in \$PATH."
|
|
echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# util::install_kubectl will install the given version kubectl
|
|
function util::install_kubectl {
|
|
local KUBECTL_VERSION=${1}
|
|
local ARCH=${2}
|
|
|
|
curl -sSL --retry 5 https://dl.k8s.io/release/"$KUBECTL_VERSION"/bin/linux/"$ARCH"/kubectl > ./kubectl
|
|
chmod +x ./kubectl
|
|
sudo rm -rf "$(which kubectl)"
|
|
sudo mv ./kubectl /usr/local/bin/kubectl
|
|
}
|
|
|
|
# util::create_signing_certkey creates a CA, args are sudo, dest-dir, ca-id, purpose
|
|
function util::create_signing_certkey {
|
|
local sudo=$1
|
|
local dest_dir=$2
|
|
local id=$3
|
|
local purpose=$4
|
|
OPENSSL_BIN=$(command -v openssl)
|
|
# Create ca
|
|
${sudo} /usr/bin/env bash -e <<EOF
|
|
rm -f "${dest_dir}/${id}-ca.crt" "${dest_dir}/${id}-ca.key"
|
|
${OPENSSL_BIN} req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "${dest_dir}/${id}-ca.key" -out "${dest_dir}/${id}-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
|
|
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment",${purpose}]}}}' > "${dest_dir}/${id}-ca-config.json"
|
|
EOF
|
|
}
|
|
|
|
# util::create_certkey signs a certificate: args are sudo, dest-dir, ca, filename (roughly), subject, hosts...
|
|
function util::create_certkey {
|
|
local sudo=$1
|
|
local dest_dir=$2
|
|
local ca=$3
|
|
local id=$4
|
|
local cn=${5:-$4}
|
|
local hosts=""
|
|
local SEP=""
|
|
shift 5
|
|
while [[ -n "${1:-}" ]]; do
|
|
hosts+="${SEP}\"$1\""
|
|
SEP=","
|
|
shift 1
|
|
done
|
|
${sudo} /usr/bin/env bash -e <<EOF
|
|
cd ${dest_dir}
|
|
echo '{"CN":"${cn}","hosts":[${hosts}],"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | ${CFSSL_BIN} gencert -ca=${ca}.crt -ca-key=${ca}.key -config=${ca}-config.json - | ${CFSSLJSON_BIN} -bare ${id}
|
|
mv "${id}-key.pem" "${id}.key"
|
|
mv "${id}.pem" "${id}.crt"
|
|
rm -f "${id}.csr"
|
|
EOF
|
|
}
|
|
|
|
# util::append_client_kubeconfig creates a new context including a cluster and a user to the existed kubeconfig file
|
|
function util::append_client_kubeconfig {
|
|
local kubeconfig_path=$1
|
|
local client_certificate_file=$2
|
|
local client_key_file=$3
|
|
local api_host=$4
|
|
local api_port=$5
|
|
local client_id=$6
|
|
local token=${7:-}
|
|
kubectl config set-cluster "${client_id}" --server=https://"${api_host}:${api_port}" --insecure-skip-tls-verify=true --kubeconfig="${kubeconfig_path}"
|
|
kubectl config set-credentials "${client_id}" --token="${token}" --client-certificate="${client_certificate_file}" --client-key="${client_key_file}" --embed-certs=true --kubeconfig="${kubeconfig_path}"
|
|
kubectl config set-context "${client_id}" --cluster="${client_id}" --user="${client_id}" --kubeconfig="${kubeconfig_path}"
|
|
}
|
|
|
|
# util::write_client_kubeconfig creates a self-contained kubeconfig: args are sudo, dest-dir, client certificate data, client key data, host, port, client id, token(optional)
|
|
function util::write_client_kubeconfig {
|
|
local sudo=$1
|
|
local dest_dir=$2
|
|
local client_certificate_data=$3
|
|
local client_key_data=$4
|
|
local api_host=$5
|
|
local api_port=$6
|
|
local client_id=$7
|
|
local token=${8:-}
|
|
cat <<EOF | ${sudo} tee "${dest_dir}"/"${client_id}".config > /dev/null
|
|
apiVersion: v1
|
|
kind: Config
|
|
clusters:
|
|
- cluster:
|
|
"insecure-skip-tls-verify": true
|
|
server: https://${api_host}:${api_port}/
|
|
name: karmada-apiserver
|
|
users:
|
|
- user:
|
|
token: ${token}
|
|
client-certificate-data: ${client_certificate_data}
|
|
client-key-data: ${client_key_data}
|
|
name: karmada-apiserver
|
|
contexts:
|
|
- context:
|
|
cluster: karmada-apiserver
|
|
user: karmada-apiserver
|
|
name: karmada-apiserver
|
|
current-context: karmada-apiserver
|
|
EOF
|
|
${sudo} chmod 0644 "${dest_dir}"/"${client_id}".config
|
|
}
|
|
|
|
# util::wait_for_condition blocks until the provided condition becomes true
|
|
# Arguments:
|
|
# - 1: message indicating what conditions is being waited for (e.g. 'ok')
|
|
# - 2: a string representing an eval'able condition. When eval'd it should not output
|
|
# anything to stdout or stderr.
|
|
# - 3: optional timeout in seconds. If not provided, waits forever.
|
|
# Returns:
|
|
# 1 if the condition is not met before the timeout
|
|
function util::wait_for_condition() {
|
|
local msg=$1
|
|
# condition should be a string that can be eval'd.
|
|
local condition=$2
|
|
local timeout=${3:-}
|
|
|
|
local start_msg="Waiting for ${msg}"
|
|
local error_msg="[ERROR] Timeout waiting for ${msg}"
|
|
|
|
local counter=0
|
|
while ! eval ${condition}; do
|
|
if [[ "${counter}" = "0" ]]; then
|
|
echo -n "${start_msg}"
|
|
fi
|
|
|
|
if [[ -z "${timeout}" || "${counter}" -lt "${timeout}" ]]; then
|
|
counter=$((counter + 1))
|
|
if [[ -n "${timeout}" ]]; then
|
|
echo -n '.'
|
|
fi
|
|
sleep 1
|
|
else
|
|
echo -e "\n${error_msg}"
|
|
return 1
|
|
fi
|
|
done
|
|
|
|
if [[ "${counter}" != "0" && -n "${timeout}" ]]; then
|
|
echo ' done'
|
|
fi
|
|
}
|
|
|
|
# util::wait_file_exist checks if a file exists, if not, wait until timeout
|
|
function util::wait_file_exist() {
|
|
local file_path=${1}
|
|
local timeout=${2}
|
|
for ((time=0; time<${timeout}; time++)); do
|
|
if [[ -e ${file_path} ]]; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# util::wait_pod_ready waits for pod state becomes ready until timeout.
|
|
# Parmeters:
|
|
# - $1: pod label, such as "app=etcd"
|
|
# - $2: pod namespace, such as "karmada-system"
|
|
# - $3: time out, such as "200s"
|
|
function util::wait_pod_ready() {
|
|
local pod_label=$1
|
|
local pod_namespace=$2
|
|
|
|
echo "wait the $pod_label ready..."
|
|
set +e
|
|
util::kubectl_with_retry wait --for=condition=Ready --timeout=200s pods -l app=${pod_label} -n ${pod_namespace}
|
|
ret=$?
|
|
set -e
|
|
return ${ret}
|
|
}
|
|
|
|
# util::kubectl_with_retry will retry if execute kubectl command failed
|
|
# tolerate kubectl command failure that may happen before the pod is created by StatefulSet/Deployment.
|
|
function util::kubectl_with_retry() {
|
|
local ret=0
|
|
for i in `seq 1 30`; do
|
|
kubectl "$@"
|
|
ret=$?
|
|
if [[ ${ret} -ne 0 ]]; then
|
|
echo "kubectl $@ failed, retrying"
|
|
sleep 1
|
|
continue
|
|
else
|
|
return 0
|
|
fi
|
|
done
|
|
|
|
echo "kubectl $@ failed"
|
|
kubectl "$@"
|
|
return ${ret}
|
|
}
|
|
|
|
# util::create_cluster creates a kubernetes cluster
|
|
# util::create_cluster creates a kind cluster and don't wait for control plane node to be ready.
|
|
# Parmeters:
|
|
# - $1: cluster name, such as "host"
|
|
# - $2: KUBECONFIG file, such as "/var/run/host.config"
|
|
# - $3: node docker image to use for booting the cluster, such as "kindest/node:v1.19.1"
|
|
# - $4: log file path, such as "/tmp/logs/"
|
|
function util::create_cluster() {
|
|
local cluster_name=${1}
|
|
local kubeconfig=${2}
|
|
local kind_image=${3}
|
|
local log_path=${4}
|
|
local cluster_config=${5:-}
|
|
|
|
mkdir -p ${log_path}
|
|
rm -rf "${log_path}/${cluster_name}.log"
|
|
rm -f "${kubeconfig}"
|
|
nohup kind delete cluster --name="${cluster_name}" >> "${log_path}"/"${cluster_name}".log 2>&1 && kind create cluster --name "${cluster_name}" --kubeconfig="${kubeconfig}" --image="${kind_image}" --config="${cluster_config}" >> "${log_path}"/"${cluster_name}".log 2>&1 &
|
|
echo "Creating cluster ${cluster_name}"
|
|
}
|
|
|
|
# util::check_clusters_ready checks if a cluster is ready, if not, wait until timeout
|
|
function util::check_clusters_ready() {
|
|
local kubeconfig_path=${1}
|
|
local context_name=${2}
|
|
|
|
echo "Waiting for kubeconfig file ${kubeconfig_path} and clusters ${context_name} to be ready..."
|
|
util::wait_file_exist "${kubeconfig_path}" 300
|
|
util::wait_for_condition 'running' "docker inspect --format='{{.State.Status}}' ${context_name}-control-plane &> /dev/null" 300
|
|
|
|
kubectl config rename-context "kind-${context_name}" "${context_name}" --kubeconfig="${kubeconfig_path}"
|
|
container_ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${context_name}-control-plane")
|
|
kubectl config set-cluster "kind-${context_name}" --server="https://${container_ip}:6443" --kubeconfig="${kubeconfig_path}"
|
|
|
|
util::wait_for_condition 'ok' "kubectl --kubeconfig ${kubeconfig_path} --context ${context_name} get --raw=/healthz &> /dev/null" 300
|
|
}
|
|
|
|
# This function deploys webhook configuration
|
|
# Parameters:
|
|
# - $1: CA file
|
|
# - $2: configuration file
|
|
# Note:
|
|
# Deprecated: should be removed after helm get on board.
|
|
function util::deploy_webhook_configuration() {
|
|
local ca_file=$1
|
|
local conf=$2
|
|
|
|
local ca_string=$(sudo cat ${ca_file} | base64 | tr "\n" " "|sed s/[[:space:]]//g)
|
|
|
|
local temp_path=$(mktemp -d)
|
|
cp -rf "${conf}" "${temp_path}/temp.yaml"
|
|
sed -i "s/{{caBundle}}/${ca_string}/g" "${temp_path}/temp.yaml"
|
|
kubectl apply -f "${temp_path}/temp.yaml"
|
|
rm -rf "${temp_path}"
|
|
}
|
|
|
|
# util::wait_service_external_ip give a service external ip when it is ready, if not, wait until timeout
|
|
# Parameters:
|
|
# - $1: service name in k8s
|
|
# - $2: namespace
|
|
SERVICE_EXTERNAL_IP=''
|
|
function util::wait_service_external_ip() {
|
|
local service_name=$1
|
|
local namespace=$2
|
|
local external_ip
|
|
local tmp
|
|
for tmp in {1..30}; do
|
|
set +e
|
|
external_ip=$(kubectl get service "${service_name}" -n "${namespace}" --template="{{range .status.loadBalancer.ingress}}{{.ip}} {{end}}" | xargs)
|
|
set -e
|
|
if [[ -z "$external_ip" ]]; then
|
|
echo "wait the external ip of ${service_name} ready..."
|
|
sleep 6
|
|
continue
|
|
else
|
|
SERVICE_EXTERNAL_IP="${external_ip}"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# util::get_load_balancer_ip get a valid load balancer ip from k8s service's 'loadBalancer' , if not, wait until timeout
|
|
# call 'util::wait_service_external_ip' before using this function
|
|
function util::get_load_balancer_ip() {
|
|
local tmp
|
|
local first_ip
|
|
if [[ -n "${SERVICE_EXTERNAL_IP}" ]]; then
|
|
first_ip=$(echo "${SERVICE_EXTERNAL_IP}" | awk '{print $1}') #temporarily choose the first one
|
|
for tmp in {1..10}; do
|
|
set +e
|
|
connect_test=$(curl -s -k -m 5 https://"${first_ip}":5443/readyz)
|
|
set -e
|
|
if [[ "${connect_test}" = "ok" ]]; then
|
|
echo "${first_ip}"
|
|
return 0
|
|
else
|
|
sleep 3
|
|
continue
|
|
fi
|
|
done
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# util::add_routes will add routes for given kind cluster
|
|
# Parameters:
|
|
# - $1: name of the kind cluster want to add routes
|
|
# - $2: the kubeconfig path of the cluster wanted to be connected
|
|
# - $3: the context in kubeconfig of the cluster wanted to be connected
|
|
function util::add_routes() {
|
|
unset IFS
|
|
routes=$(kubectl --kubeconfig ${2} --context ${3} get nodes -o jsonpath='{range .items[*]}ip route add {.spec.podCIDR} via {.status.addresses[?(.type=="InternalIP")].address}{"\n"}')
|
|
echo "Connecting cluster ${1} to ${2}"
|
|
|
|
IFS=$'\n'
|
|
for n in $(kind get nodes --name "${1}"); do
|
|
for r in $routes; do
|
|
echo "exec cmd in docker $n $r"
|
|
eval "docker exec $n $r"
|
|
done
|
|
done
|
|
unset IFS
|
|
}
|
|
|