Merge pull request #5305 from chaosi-zju/helm-0627

fix controller can't restart in helm for dependent secret not found

charts/karmada/templates/_helpers.tpl

@@ -586,40 +586,31 @@ Return the proper Docker Image Registry Secret Names
 {{- end }}
 {{- end -}}
 
-{{- define "karmada.init-sa-secret.volume" -}}
-{{- $name := include "karmada.name" . -}}
-- name: init-sa-secret
-  secret:
-    secretName: {{ $name }}-hook-job
-{{- end -}}
-
-{{- define "karmada.init-sa-secret.volumeMount" -}}
-- name: init-sa-secret
-  mountPath: /opt/mount
-{{- end -}}
-
-{{- define "karmada.initContainer.build-kubeconfig" -}}
-TOKEN=$(cat /opt/mount/token)
-kubectl config set-cluster karmada-host --server=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --certificate-authority=/opt/mount/ca.crt
-kubectl config set-credentials default --token=$TOKEN
-kubectl config set-context karmada-host-context --cluster=karmada-host --user=default --namespace=default
-kubectl config use-context karmada-host-context
-{{- end -}}
-
 {{- define "karmada.initContainer.waitEtcd" -}}
 - name: wait
-  image: {{ include "karmada.kubectl.image" . }}
+  image: {{ include "karmada.cfssl.image" . }}
   imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }}
   command:
     - /bin/sh
     - -c
     - |
       bash <<'EOF'
-      {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }}
-      kubectl rollout status statefulset etcd -n {{ include "karmada.namespace" . }}
+      set -ex
+      while true; do
+        ETCD_ENDPOINT=${ETCD_CLIENT_SERVICE_HOST}":"${ETCD_CLIENT_SERVICE_PORT}
+
+        # check etcd connectivity by executing curl.
+        # if etcd is ready, the response of curl would be `curl: (52) Empty reply from server`, with return code 52.
+        # if not, the response of curl would be like `curl: (7) Failed to connect to .....`, with other return code.
+        if curl --connect-timeout 2 ${ETCD_ENDPOINT} || [ $? -eq 52 ]; then
+          break
+        fi
+
+        echo "failed to connect to "${ETCD_ENDPOINT}
+        sleep 2
+      done
+      echo "successfully connect to "${ETCD_ENDPOINT}
       EOF
-  volumeMounts:
-    {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }}
 {{- end -}}
 
 {{- define "karmada.initContainer.waitStaticResource" -}}
@@ -631,9 +622,18 @@ kubectl config use-context karmada-host-context
     - -c
     - |
       bash <<'EOF'
-      {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }}
-      kubectl wait --for=condition=complete job {{ include "karmada.name" . }}-static-resource -n {{ include "karmada.namespace" . }}
+      set -ex
+
+      # here are three cases:
+      # case first installation: no `cm/karmada-version` at first, so when you get it, it means `karmada-static-resource-job` finished.
+      # case restart: already has `cm/karmada-version`, which means `karmada-static-resource-job` already finished.
+      # case upgrading: already has `cm/karmada-version`, but it may be old version, we should wait until `.data.karmadaVersion` equal to current `.Values.karmadaImageVersion`.
+      while [[ $(kubectl --kubeconfig /etc/kubeconfig get configmap karmada-version -n {{ .Values.systemNamespace }} -o jsonpath='{.data.karmadaVersion}') != {{ .Values.karmadaImageVersion }} ]]; do
+        echo "wait for karmada-static-resource-job finished"; sleep 2
+      done
+
+      echo "karmada-static-resource-job successfully completed since expected configmap value was found"
       EOF
   volumeMounts:
-    {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }}
+    {{- include "karmada.kubeconfig.volumeMount" .| nindent 4 }}
 {{- end -}}

charts/karmada/templates/karmada-aggregated-apiserver.yaml

@@ -98,7 +98,6 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:

charts/karmada/templates/karmada-apiserver.yaml

@@ -137,7 +137,6 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:
             secretName: {{ $name }}-cert

charts/karmada/templates/karmada-controller-manager.yaml

@@ -42,7 +42,6 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
       initContainers:
         {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}

charts/karmada/templates/karmada-descheduler.yaml

@@ -78,7 +78,6 @@ spec:
           resources:
           {{- toYaml .Values.descheduler.resources | nindent 12 }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.descheduler.kubeconfig.volume" . | nindent 8 }}
       {{- include "karmada.scheduler.cert.volume" . | nindent 8 }}
 

charts/karmada/templates/karmada-metrics-adapter.yaml

@@ -83,7 +83,6 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:

charts/karmada/templates/karmada-scheduler.yaml

@@ -76,7 +76,6 @@ spec:
           resources:
           {{- toYaml .Values.scheduler.resources | nindent 12 }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
       {{- include "karmada.scheduler.cert.volume" . | nindent 8 }}
 

charts/karmada/templates/karmada-search.yaml

@@ -92,7 +92,6 @@ spec:
           resources:
           {{- toYaml .Values.apiServer.resources | nindent 12 }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.search.kubeconfig.volume" . | nindent 8 }}
       {{- include "karmada.search.etcd.cert.volume" . | nindent 8 }}
 ---

charts/karmada/templates/karmada-static-resource-job.yaml

@@ -42,6 +42,17 @@ spec:
           kubectl apply -k /crds --kubeconfig /etc/kubeconfig
           kubectl apply -f /static-resources/system-namespace.yaml --kubeconfig /etc/kubeconfig
           kubectl apply -f /static-resources/ --kubeconfig /etc/kubeconfig
+
+          kubectl --kubeconfig /etc/kubeconfig apply -f - <<InnerEOF
+          apiVersion: v1
+          kind: ConfigMap
+          metadata:
+            name: karmada-version
+            namespace: {{ .Values.systemNamespace }}
+          data:
+            karmadaVersion: {{ .Values.karmadaImageVersion }}
+          InnerEOF
+
           EOF
         volumeMounts:
           - name: {{ $name }}-crds-kustomization

charts/karmada/templates/karmada-webhook.yaml

@@ -71,7 +71,6 @@ spec:
           resources:
           {{- toYaml .Values.webhook.resources | nindent 12 }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: {{ $name }}-webhook-cert-secret
           secret:

charts/karmada/templates/kube-controller-manager.yaml

@@ -89,7 +89,6 @@ spec:
         - name: apisever-cert
           secret:
             secretName: {{ $name }}-cert
-        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
 
 {{ if .Values.kubeControllerManager.podDisruptionBudget }}

charts/karmada/templates/post-install-job.yaml

@@ -59,6 +59,5 @@ spec:
           done
 
           kubectl delete job {{ $name }}-static-resource -n {{ $namespace }}
-          kubectl delete secret {{ $name }}-hook-job -n {{ $namespace }}
           EOF
 {{- end }}

charts/karmada/templates/pre-install-job.yaml

@@ -459,21 +459,6 @@ metadata:
     {{- include "karmada.preInstallJob.labels" . | nindent 4 }}
   {{- end }}
 ---
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ $name }}-hook-job
-  namespace: {{ $namespace }}
-  annotations:
-    "kubernetes.io/service-account.name": {{ $name }}-hook-job
-    "helm.sh/hook": pre-install
-    "helm.sh/hook-weight": "1"
-    {{- if "karmada.preInstallJob.labels" }}
-    labels:
-      {{- include "karmada.preInstallJob.labels" . | nindent 4 }}
-    {{- end }}
-type: kubernetes.io/service-account-token
----
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata: