support install karmada aggregated apiserver by helm

Signed-off-by: jrkeen <jrkeen@hotmail.com>

charts/README.md

@@ -112,6 +112,9 @@ $ helm install karmada-scheduler-estimator -n karmada-system ./charts
 |`certs.custom.caCrt`|CA CRT of the certificate|`""`|
 |`certs.custom.crt`|CRT of the certificate|`""`|
 |`certs.custom.key`|KEY of the certificate|`""`|
+|`certs.custom.frontProxyCaCrt`|CA CRT of the front proxy certificate|`""`|
+|`certs.custom.frontProxyCrt`|CRT of the front proxy certificate|`""`|
+|`certs.custom.frontProxyKey`|KEY of the front proxy certificate|`""`|
 |`etcd.mode`| Mode "external" and "internal" are provided, "external" means use external ectd, "internal" means install a etcd in the cluster |`"internal"`|
 |`etcd.external.servers`| Servers of etcd |`""`|
 |`etcd.external.registryPrefix`| Use to registry prefix of etcd |`"/registry/karmada"`|
@@ -194,6 +197,18 @@ $ helm install karmada-scheduler-estimator -n karmada-system ./charts
 |`apiServer.tolerations`| Tolerations of the karmada-apiserver |`[]`|
 |`apiServer.serviceType`| Service type of apiserver, accepts "ClusterIP", "NodePort", "LoadBalancer" |`"ClusterIP"`|
 |`apiServer.nodePort`| Node port for apiserver, takes effect when `apiServer.serviceType` is "NodePort". If no port is specified, the nodePort will be automatically assigned. |`0`|
+|`aggregatedApiServer.labels`| Labels of the karmada-aggregated-apiserver deployment |`{"app": "karmada-aggregated-apiserver"}`|
+|`aggregatedApiServer.replicaCount`| Target replicas of the karmada-aggregated-apiserver |`1`|
+|`aggregatedApiServer.podLabels`| Labels of the karmada-aggregated-apiserver pods |`{}`|
+|`aggregatedApiServer.podAnnotations`| Annotaions of the karmada-aggregated-apiserver pods |`{}`|
+|`aggregatedApiServer.imagePullSecrets`| Image pull secret of the karmada-aggregated-apiserver |`[]`|
+|`aggregatedApiServer.image.repository`| Image of the karmada-aggregated-apiserver |`"swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-aggregated-apiserver"`|
+|`aggregatedApiServer.image.tag`| Image tag of the karmada-aggregated-apiserver |`"latest"`|
+|`aggregatedApiServer.image.pullPolicy`| Image pull policy of the karmada-aggregated-apiserver |`"IfNotPresent"`|
+|`aggregatedApiServer.resources`| Resource quota of the karmada-aggregated-apiserver |`{requests: {cpu: 100m}}`|
+|`aggregatedApiServer.nodeSelector`| Node selector of the karmada-aggregated-apiserver |`{}`|
+|`aggregatedApiServer.affinity`| Affinity of the karmada-aggregated-apiserver |`{}`|
+|`aggregatedApiServer.tolerations`| Tolerations of the karmada-aggregated-apiserver |`[]`|
 |`kubeControllerManager.labels`| Labels of the kube-controller-manager deployment |`{"app": "kube-controller-manager"}`|
 |`kubeControllerManager.replicaCount`| Target replicas of the kube-controller-manager |`1`|
 |`kubeControllerManager.podLabels`| Labels of the kube-controller-manager pods |`{}`|

charts/templates/_helpers.tpl

@@ -26,6 +26,24 @@ app: {{- include "karmada.name" .}}-apiserver
 {{- end }}
 {{- end -}}
 
+{{- define "karmada.aggregatedApiserver.labels" -}}
+{{- if .Values.aggregatedApiServer.labels }}
+{{- range $key, $value := .Values.aggregatedApiServer.labels}}
+{{ $key }}: {{ $value }}
+{{- end}}
+{{- else}}
+app: {{- include "karmada.name" .}}-aggregated-apiserver
+{{- end }}
+{{- end -}}
+
+{{- define "karmada.aggregatedApiserver.podLabels" -}}
+{{- if .Values.aggregatedApiServer.podLabels }}
+{{- range $key, $value := .Values.aggregatedApiServer.podLabels}}
+{{ $key }}: {{ $value }}
+{{- end}}
+{{- end }}
+{{- end -}}
+
 {{- define "karmada.kube-cm.labels" -}}
 {{- if .Values.kubeControllerManager.labels }}
 {{- range $key, $value := .Values.kubeControllerManager.labels}}

charts/templates/_karmada_apiservice.tpl

@@ -0,0 +1,30 @@
+{{- define "karmada.apiservice" -}}
+{{- $name := include "karmada.name" . -}}
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1alpha1.cluster.karmada.io
+  labels:
+    app: {{ $name }}-aggregated-apiserver
+    apiserver: "true"
+spec:
+  insecureSkipTLSVerify: true
+  group: cluster.karmada.io
+  groupPriorityMinimum: 2000
+  service:
+    name: {{ $name }}-aggregated-apiserver
+    namespace: {{ include "karmada.namespace" . }}
+  version: v1alpha1
+  versionPriority: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $name }}-aggregated-apiserver
+  namespace: {{ include "karmada.namespace" . }}
+spec:
+  type: ExternalName
+  externalName: {{ $name }}-aggregated-apiserver.{{ include "karmada.namespace" . }}.svc.{{ .Values.clusterDomain }}
+---
+{{- end -}}

charts/templates/_karmada_system_namespace.tpl

@@ -3,7 +3,7 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: karmada-system
+  name: {{ include "karmada.namespace" . }}
 ---
 apiVersion: v1
 kind: Namespace

charts/templates/_karmada_webhook_configuration.tpl

@@ -60,7 +60,7 @@ webhooks:
         scope: "Cluster"
     clientConfig:
       url: https://karmada-webhook.karmada-system.svc:443/validate-clusteroverridepolicy
-      caBundle: {{caBundle}}
+      {{- include "karmada.webhook.caBundle" . | nindent 6 }}
     failurePolicy: Fail
     sideEffects: None
     admissionReviewVersions: ["v1"]

charts/templates/karmada-aggregated-apiserver.yaml

@@ -0,0 +1,108 @@
+{{- if eq .Values.installMode "host" }}
+{{- $name := include "karmada.name" . -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $name }}-aggregated-apiserver
+  namespace: {{ include "karmada.namespace" . }}
+  labels:
+    {{- include "karmada.aggregatedApiserver.labels" . | nindent 4}}
+spec:
+  selector:
+    matchLabels:
+      {{- include "karmada.aggregatedApiserver.labels" . | nindent 6}}
+  replicas: {{ .Values.aggregatedApiServer.replicaCount }}
+  template:
+    metadata:
+      {{- with .Values.aggregatedApiServer.podAnnotations }}
+      annotations:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "karmada.aggregatedApiserver.labels" . | nindent 8}}
+        {{- include "karmada.aggregatedApiserver.podLabels" . | nindent 8}}
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: {{ $name }}-aggregated-apiserver
+          image: "{{ .Values.aggregatedApiServer.image.repository }}:{{ .Values.aggregatedApiServer.image.tag | default "latest" }}"
+          imagePullPolicy: {{ .Values.aggregatedApiServer.image.pullPolicy }}
+          volumeMounts:
+            {{- include "karmada.kubeconfig.volumeMount" . | nindent 12}}
+            - name: etcd-cert
+              mountPath: /etc/etcd/pki
+              readOnly: true
+            - name: apiserver-cert
+              mountPath: /etc/kubernetes/pki
+              readOnly: true
+          command:
+            - /bin/karmada-aggregated-apiserver
+            - --kubeconfig=/etc/kubeconfig
+            - --authentication-kubeconfig=/etc/kubeconfig
+            - --authorization-kubeconfig=/etc/kubeconfig
+            - --karmada-config=/etc/kubeconfig
+            {{- if eq .Values.etcd.mode "external" }}
+            - --etcd-cafile=/etc/etcd/pki/ca.crt
+            - --etcd-certfile=/etc/etcd/pki/tls.crt
+            - --etcd-keyfile=/etc/etcd/pki/tls.key
+            - --etcd-servers={{ .Values.etcd.external.servers }}
+            - --etcd-prefix={{ .Values.etcd.external.registryPrefix }}
+            {{- end  }}
+            {{- if eq .Values.etcd.mode "internal" }}
+            - --etcd-cafile=/etc/etcd/pki/server-ca.crt
+            - --etcd-certfile=/etc/etcd/pki/karmada.crt
+            - --etcd-keyfile=/etc/etcd/pki/karmada.key
+            - --etcd-servers=https://etcd-client.{{ include "karmada.namespace" . }}.svc.{{ .Values.clusterDomain }}:2379
+            {{- end  }}
+            - --tls-cert-file=/etc/kubernetes/pki/karmada.crt
+            - --tls-private-key-file=/etc/kubernetes/pki/karmada.key
+            - --audit-log-path=-
+            - --feature-gates=APIPriorityAndFairness=false
+            - --audit-log-maxage=0
+            - --audit-log-maxbackup=0
+          resources:
+            {{- toYaml .Values.aggregatedApiServer.resources | nindent 12 }}
+      {{- with .Values.aggregatedApiServer.nodeSelector }}
+      nodeSelector:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.aggregatedApiServer.affinity }}
+      affinity:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.aggregatedApiServer.tolerations }}
+      tolerations:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        {{- include "karmada.kubeconfig.volume" . | nindent 8}}
+        - name: apiserver-cert
+          secret:
+            secretName: {{ $name }}-cert
+        - name: etcd-cert
+          secret:
+          {{- if eq .Values.etcd.mode "internal" }}
+            secretName: {{ $name }}-cert
+          {{- end }}
+          {{- if eq .Values.etcd.mode "external" }}
+            secretName: external-etcd-cert
+          {{- end }}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $name }}-aggregated-apiserver
+  namespace: {{ include "karmada.namespace" . }}
+  labels:
+    {{- include "karmada.aggregatedApiserver.labels" . | nindent 4}}
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 443
+  selector:
+    {{- include "karmada.aggregatedApiserver.labels" . | nindent 4}}
+
+{{- end}}

charts/templates/karmada_apiserver.yaml

@@ -61,10 +61,10 @@ spec:
             - --service-account-key-file=/etc/kubernetes/pki/karmada.key
             - --service-account-signing-key-file=/etc/kubernetes/pki/karmada.key
             - --service-cluster-ip-range=10.96.0.0/12
-            - --proxy-client-cert-file=/etc/kubernetes/pki/karmada.crt
+            - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
-            - --proxy-client-key-file=/etc/kubernetes/pki/karmada.key
+            - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
             - --requestheader-allowed-names=front-proxy-client
-            - --requestheader-client-ca-file=/etc/kubernetes/pki/server-ca.crt
+            - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
             - --requestheader-extra-headers-prefix=X-Remote-Extra-
             - --requestheader-group-headers=X-Remote-Group
             - --requestheader-username-headers=X-Remote-User

charts/templates/karmada_cert.yaml

@@ -12,6 +12,12 @@ data:
     {{ b64enc .Values.certs.custom.crt }}
   karmada.key: |
     {{ b64enc .Values.certs.custom.key }}
+  front-proxy-ca.crt: |
+    {{ b64enc .Values.certs.custom.frontProxyCaCrt }}
+  front-proxy-client.crt: |
+    {{ b64enc .Values.certs.custom.frontProxyCrt }}
+  front-proxy-client.key: |
+    {{ b64enc .Values.certs.custom.frontProxyKey }}
 ---
 apiVersion: v1
 kind: Secret

charts/templates/pre-install-job.yaml

@@ -24,6 +24,12 @@ data:
         {{ print "{{ crt }}" }}
       karmada.key: |-
         {{ print "{{ key }}" }}
+      front-proxy-ca.crt: |-
+        {{ print "{{ front_proxy_ca_crt }}" }}
+      front-proxy-client.crt: |-
+        {{ print "{{ front_proxy_crt }}" }}
+      front-proxy-client.key: |-
+        {{ print "{{ front_proxy_key }}" }}
   webhook-cert.yaml: |-
     apiVersion: v1
     kind: Secret
@@ -74,6 +80,8 @@ data:
         {{- include "karmada.webhook.configuration" . | nindent 8 }}
       {{- print "system-namespace.yaml: " | nindent 6 }} |-
         {{- include "karmada.systemNamespace" . | nindent 8 }}
+      {{- print "apiservice.yaml: " | nindent 6 }} |-
+        {{- include "karmada.apiservice" . | nindent 8 }}
   crds-configmaps.yaml: |-
     apiVersion: v1
     kind: ConfigMap
@@ -150,14 +158,23 @@ spec:
           mkdir -p /opt/certs
           cp -r -L /opt/mount/* /opt/configs/
           openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "/opt/certs/server-ca.key" -out "/opt/certs/server-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
+          openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "/opt/certs/front-proxy-ca.key" -out "/opt/certs/front-proxy-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
           echo '{"signing":{"default":{"expiry":{{ printf `"%s"` .Values.certs.auto.expiry }},"usages":["signing","key encipherment","client auth","server auth"]}}}' > "/opt/certs/server-ca-config.json"
           echo '{"CN":"system:admin","hosts":{{ tpl (toJson .Values.certs.auto.hosts) . }},"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=/opt/certs/server-ca.crt -ca-key=/opt/certs/server-ca.key -config=/opt/certs/server-ca-config.json - | cfssljson -bare /opt/certs/karmada
+          echo '{"signing":{"default":{"expiry":{{ printf `"%s"` .Values.certs.auto.expiry }},"usages":["signing","key encipherment","client auth","server auth"]}}}' > "/opt/certs/front-proxy-ca-config.json"
+          echo '{"CN":"front-proxy-client","hosts":{{ tpl (toJson .Values.certs.auto.hosts) . }},"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=/opt/certs/front-proxy-ca.crt -ca-key=/opt/certs/front-proxy-ca.key -config=/opt/certs/front-proxy-ca-config.json - | cfssljson -bare /opt/certs/front-proxy-client
           karmada_ca=$(base64 /opt/certs/server-ca.crt | tr -d '\r\n')
           karmada_crt=$(base64 /opt/certs/karmada.pem | tr -d '\r\n')
           karmada_key=$(base64 /opt/certs/karmada-key.pem | tr -d '\r\n')
+          front_proxy_ca=$(base64 /opt/certs/front-proxy-ca.crt | tr -d '\r\n')
+          front_proxy_client_crt=$(base64 /opt/certs/front-proxy-client.pem | tr -d '\r\n')
+          front_proxy_client_key=$(base64 /opt/certs/front-proxy-client-key.pem | tr -d '\r\n')
           sed -i'' -e "s/{{ print "{{ ca_crt }}" }}/${karmada_ca}/g" /opt/configs/cert.yaml
           sed -i'' -e "s/{{ print "{{ crt }}" }}/${karmada_crt}/g" /opt/configs/cert.yaml
           sed -i'' -e "s/{{ print "{{ key }}" }}/${karmada_key}/g" /opt/configs/cert.yaml
+          sed -i'' -e "s/{{ print "{{ front_proxy_ca_crt }}" }}/${front_proxy_ca}/g" /opt/configs/cert.yaml
+          sed -i'' -e "s/{{ print "{{ front_proxy_crt }}" }}/${front_proxy_client_crt}/g" /opt/configs/cert.yaml
+          sed -i'' -e "s/{{ print "{{ front_proxy_key }}" }}/${front_proxy_client_key}/g" /opt/configs/cert.yaml
           sed -i'' -e "s/{{ print "{{ ca_crt }}" }}/${karmada_ca}/g" /opt/configs/kubeconfig.yaml
           sed -i'' -e "s/{{ print "{{ crt }}" }}/${karmada_crt}/g" /opt/configs/kubeconfig.yaml
           sed -i'' -e "s/{{ print "{{ key }}" }}/${karmada_key}/g" /opt/configs/kubeconfig.yaml

charts/values.yaml

@@ -38,8 +38,8 @@ certs:
     ## @param certs.auto.hosts hosts of the certificate
     hosts: [
       "kubernetes.default.svc",
-      "*.etcd.{{ .Release.Namespace }}.svc.cluster.local",
+      "*.etcd.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}",
-      "*.{{ .Release.Namespace }}.svc.cluster.local",
+      "*.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}",
       "*.{{ .Release.Namespace }}.svc",
       "localhost",
       "127.0.0.1"
@@ -60,6 +60,21 @@ certs:
       -----BEGIN RSA PRIVATE KEY-----
       XXXXXXXXXXXXXXXXXXXXXXXXXXX
       -----END RSA PRIVATE KEY-----
+    ## @param certs.custom.frontProxyCaCrt ca of the front proxy certificate
+    frontProxyCaCrt: |
+      -----BEGIN CERTIFICATE-----
+      XXXXXXXXXXXXXXXXXXXXXXXXXXX
+      -----END CERTIFICATE-----
+    ## @param certs.custom.frontProxyCrt crt of the front proxy certificate
+    frontProxyCrt: |
+      -----BEGIN CERTIFICATE-----
+      XXXXXXXXXXXXXXXXXXXXXXXXXXX
+      -----END CERTIFICATE-----
+    ## @param certs.custom.frontProxyKey key of the front proxy certificate
+    frontProxyKey: |
+      -----BEGIN RSA PRIVATE KEY-----
+      XXXXXXXXXXXXXXXXXXXXXXXXXXX
+      -----END RSA PRIVATE KEY-----
 
 ## scheduler config
 scheduler:
@@ -230,6 +245,47 @@ apiServer:
   ## If no port is specified, the nodePort will be automatically assigned.
   nodePort: 0
 
+## karmada aggregated apiserver config
+aggregatedApiServer:
+  ## @param aggregatedApiServer.labels
+  labels: 
+    app: karmada-aggregated-apiserver
+  ## @param aggregatedApiServer.replicaCount target replicas
+  replicaCount: 1
+  ## @param aggregatedApiServer.podAnnotations
+  podAnnotations: { }
+  ## @param aggregatedApiServer.podLabels
+  podLabels: { }
+  ## @param aggregatedApiServer.imagePullSecrets
+  imagePullSecrets: []
+  image:
+    ## @param aggregatedApiServer.image.repository image of the apiserver
+    repository: swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-aggregated-apiserver
+    ## @param aggregatedApiServer.image.pullPolicy pull policy of image
+    pullPolicy: IfNotPresent
+    ## @param aggregatedApiServer.image.tag overrides the image tag whose default is the latest
+    tag: latest
+  ## @param aggregatedApiServer.resources
+  resources: 
+    requests:
+      cpu: 100m
+    # If you do want to specify resources, uncomment the following
+    # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    # limits:
+    #   cpu: 100m
+    #   memory: 128Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 128Mi
+  ## @param aggregatedApiServer.nodeSelector
+  nodeSelector: { }
+  ## @param aggregatedApiServer.affinity
+  affinity: { }
+  ## @param aggregatedApiServer.tolerations
+  tolerations: [ ]
+    # - key: node-role.kubernetes.io/master
+    #   operator: Exists
+
 ## kubernetes controller manager config
 kubeControllerManager:
   ## @param kubeControllerManager.labels