diff --git a/charts/README.md b/charts/README.md index 237036ac3..69a537ba5 100644 --- a/charts/README.md +++ b/charts/README.md @@ -112,6 +112,9 @@ $ helm install karmada-scheduler-estimator -n karmada-system ./charts |`certs.custom.caCrt`|CA CRT of the certificate|`""`| |`certs.custom.crt`|CRT of the certificate|`""`| |`certs.custom.key`|KEY of the certificate|`""`| +|`certs.custom.frontProxyCaCrt`|CA CRT of the front proxy certificate|`""`| +|`certs.custom.frontProxyCrt`|CRT of the front proxy certificate|`""`| +|`certs.custom.frontProxyKey`|KEY of the front proxy certificate|`""`| |`etcd.mode`| Mode "external" and "internal" are provided, "external" means use external ectd, "internal" means install a etcd in the cluster |`"internal"`| |`etcd.external.servers`| Servers of etcd |`""`| |`etcd.external.registryPrefix`| Use to registry prefix of etcd |`"/registry/karmada"`| @@ -194,6 +197,18 @@ $ helm install karmada-scheduler-estimator -n karmada-system ./charts |`apiServer.tolerations`| Tolerations of the karmada-apiserver |`[]`| |`apiServer.serviceType`| Service type of apiserver, accepts "ClusterIP", "NodePort", "LoadBalancer" |`"ClusterIP"`| |`apiServer.nodePort`| Node port for apiserver, takes effect when `apiServer.serviceType` is "NodePort". If no port is specified, the nodePort will be automatically assigned. |`0`| +|`aggregatedApiServer.labels`| Labels of the karmada-aggregated-apiserver deployment |`{"app": "karmada-aggregated-apiserver"}`| +|`aggregatedApiServer.replicaCount`| Target replicas of the karmada-aggregated-apiserver |`1`| +|`aggregatedApiServer.podLabels`| Labels of the karmada-aggregated-apiserver pods |`{}`| +|`aggregatedApiServer.podAnnotations`| Annotaions of the karmada-aggregated-apiserver pods |`{}`| +|`aggregatedApiServer.imagePullSecrets`| Image pull secret of the karmada-aggregated-apiserver |`[]`| +|`aggregatedApiServer.image.repository`| Image of the karmada-aggregated-apiserver |`"swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-aggregated-apiserver"`| +|`aggregatedApiServer.image.tag`| Image tag of the karmada-aggregated-apiserver |`"latest"`| +|`aggregatedApiServer.image.pullPolicy`| Image pull policy of the karmada-aggregated-apiserver |`"IfNotPresent"`| +|`aggregatedApiServer.resources`| Resource quota of the karmada-aggregated-apiserver |`{requests: {cpu: 100m}}`| +|`aggregatedApiServer.nodeSelector`| Node selector of the karmada-aggregated-apiserver |`{}`| +|`aggregatedApiServer.affinity`| Affinity of the karmada-aggregated-apiserver |`{}`| +|`aggregatedApiServer.tolerations`| Tolerations of the karmada-aggregated-apiserver |`[]`| |`kubeControllerManager.labels`| Labels of the kube-controller-manager deployment |`{"app": "kube-controller-manager"}`| |`kubeControllerManager.replicaCount`| Target replicas of the kube-controller-manager |`1`| |`kubeControllerManager.podLabels`| Labels of the kube-controller-manager pods |`{}`| diff --git a/charts/templates/_helpers.tpl b/charts/templates/_helpers.tpl index 4b9714a11..41fd25042 100644 --- a/charts/templates/_helpers.tpl +++ b/charts/templates/_helpers.tpl @@ -26,6 +26,24 @@ app: {{- include "karmada.name" .}}-apiserver {{- end }} {{- end -}} +{{- define "karmada.aggregatedApiserver.labels" -}} +{{- if .Values.aggregatedApiServer.labels }} +{{- range $key, $value := .Values.aggregatedApiServer.labels}} +{{ $key }}: {{ $value }} +{{- end}} +{{- else}} +app: {{- include "karmada.name" .}}-aggregated-apiserver +{{- end }} +{{- end -}} + +{{- define "karmada.aggregatedApiserver.podLabels" -}} +{{- if .Values.aggregatedApiServer.podLabels }} +{{- range $key, $value := .Values.aggregatedApiServer.podLabels}} +{{ $key }}: {{ $value }} +{{- end}} +{{- end }} +{{- end -}} + {{- define "karmada.kube-cm.labels" -}} {{- if .Values.kubeControllerManager.labels }} {{- range $key, $value := .Values.kubeControllerManager.labels}} diff --git a/charts/templates/_karmada_apiservice.tpl b/charts/templates/_karmada_apiservice.tpl new file mode 100644 index 000000000..3a178a69a --- /dev/null +++ b/charts/templates/_karmada_apiservice.tpl @@ -0,0 +1,30 @@ +{{- define "karmada.apiservice" -}} +{{- $name := include "karmada.name" . -}} +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1alpha1.cluster.karmada.io + labels: + app: {{ $name }}-aggregated-apiserver + apiserver: "true" +spec: + insecureSkipTLSVerify: true + group: cluster.karmada.io + groupPriorityMinimum: 2000 + service: + name: {{ $name }}-aggregated-apiserver + namespace: {{ include "karmada.namespace" . }} + version: v1alpha1 + versionPriority: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-aggregated-apiserver + namespace: {{ include "karmada.namespace" . }} +spec: + type: ExternalName + externalName: {{ $name }}-aggregated-apiserver.{{ include "karmada.namespace" . }}.svc.{{ .Values.clusterDomain }} +--- +{{- end -}} diff --git a/charts/templates/_karmada_system_namespace.tpl b/charts/templates/_karmada_system_namespace.tpl index ce91705c7..f65ddf9e0 100644 --- a/charts/templates/_karmada_system_namespace.tpl +++ b/charts/templates/_karmada_system_namespace.tpl @@ -3,7 +3,7 @@ apiVersion: v1 kind: Namespace metadata: - name: karmada-system + name: {{ include "karmada.namespace" . }} --- apiVersion: v1 kind: Namespace diff --git a/charts/templates/_karmada_webhook_configuration.tpl b/charts/templates/_karmada_webhook_configuration.tpl index 58ec2a313..eae965b67 100644 --- a/charts/templates/_karmada_webhook_configuration.tpl +++ b/charts/templates/_karmada_webhook_configuration.tpl @@ -60,7 +60,7 @@ webhooks: scope: "Cluster" clientConfig: url: https://karmada-webhook.karmada-system.svc:443/validate-clusteroverridepolicy - caBundle: {{caBundle}} + {{- include "karmada.webhook.caBundle" . | nindent 6 }} failurePolicy: Fail sideEffects: None admissionReviewVersions: ["v1"] diff --git a/charts/templates/karmada-aggregated-apiserver.yaml b/charts/templates/karmada-aggregated-apiserver.yaml new file mode 100644 index 000000000..e34b34023 --- /dev/null +++ b/charts/templates/karmada-aggregated-apiserver.yaml @@ -0,0 +1,108 @@ +{{- if eq .Values.installMode "host" }} +{{- $name := include "karmada.name" . -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }}-aggregated-apiserver + namespace: {{ include "karmada.namespace" . }} + labels: + {{- include "karmada.aggregatedApiserver.labels" . | nindent 4}} +spec: + selector: + matchLabels: + {{- include "karmada.aggregatedApiserver.labels" . | nindent 6}} + replicas: {{ .Values.aggregatedApiServer.replicaCount }} + template: + metadata: + {{- with .Values.aggregatedApiServer.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "karmada.aggregatedApiserver.labels" . | nindent 8}} + {{- include "karmada.aggregatedApiserver.podLabels" . | nindent 8}} + spec: + automountServiceAccountToken: false + containers: + - name: {{ $name }}-aggregated-apiserver + image: "{{ .Values.aggregatedApiServer.image.repository }}:{{ .Values.aggregatedApiServer.image.tag | default "latest" }}" + imagePullPolicy: {{ .Values.aggregatedApiServer.image.pullPolicy }} + volumeMounts: + {{- include "karmada.kubeconfig.volumeMount" . | nindent 12}} + - name: etcd-cert + mountPath: /etc/etcd/pki + readOnly: true + - name: apiserver-cert + mountPath: /etc/kubernetes/pki + readOnly: true + command: + - /bin/karmada-aggregated-apiserver + - --kubeconfig=/etc/kubeconfig + - --authentication-kubeconfig=/etc/kubeconfig + - --authorization-kubeconfig=/etc/kubeconfig + - --karmada-config=/etc/kubeconfig + {{- if eq .Values.etcd.mode "external" }} + - --etcd-cafile=/etc/etcd/pki/ca.crt + - --etcd-certfile=/etc/etcd/pki/tls.crt + - --etcd-keyfile=/etc/etcd/pki/tls.key + - --etcd-servers={{ .Values.etcd.external.servers }} + - --etcd-prefix={{ .Values.etcd.external.registryPrefix }} + {{- end }} + {{- if eq .Values.etcd.mode "internal" }} + - --etcd-cafile=/etc/etcd/pki/server-ca.crt + - --etcd-certfile=/etc/etcd/pki/karmada.crt + - --etcd-keyfile=/etc/etcd/pki/karmada.key + - --etcd-servers=https://etcd-client.{{ include "karmada.namespace" . }}.svc.{{ .Values.clusterDomain }}:2379 + {{- end }} + - --tls-cert-file=/etc/kubernetes/pki/karmada.crt + - --tls-private-key-file=/etc/kubernetes/pki/karmada.key + - --audit-log-path=- + - --feature-gates=APIPriorityAndFairness=false + - --audit-log-maxage=0 + - --audit-log-maxbackup=0 + resources: + {{- toYaml .Values.aggregatedApiServer.resources | nindent 12 }} + {{- with .Values.aggregatedApiServer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.aggregatedApiServer.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.aggregatedApiServer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- include "karmada.kubeconfig.volume" . | nindent 8}} + - name: apiserver-cert + secret: + secretName: {{ $name }}-cert + - name: etcd-cert + secret: + {{- if eq .Values.etcd.mode "internal" }} + secretName: {{ $name }}-cert + {{- end }} + {{- if eq .Values.etcd.mode "external" }} + secretName: external-etcd-cert + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-aggregated-apiserver + namespace: {{ include "karmada.namespace" . }} + labels: + {{- include "karmada.aggregatedApiserver.labels" . | nindent 4}} +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + {{- include "karmada.aggregatedApiserver.labels" . | nindent 4}} + +{{- end}} diff --git a/charts/templates/karmada_apiserver.yaml b/charts/templates/karmada_apiserver.yaml index e89a36141..b77f3d98d 100644 --- a/charts/templates/karmada_apiserver.yaml +++ b/charts/templates/karmada_apiserver.yaml @@ -61,10 +61,10 @@ spec: - --service-account-key-file=/etc/kubernetes/pki/karmada.key - --service-account-signing-key-file=/etc/kubernetes/pki/karmada.key - --service-cluster-ip-range=10.96.0.0/12 - - --proxy-client-cert-file=/etc/kubernetes/pki/karmada.crt - - --proxy-client-key-file=/etc/kubernetes/pki/karmada.key + - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt + - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key - --requestheader-allowed-names=front-proxy-client - - --requestheader-client-ca-file=/etc/kubernetes/pki/server-ca.crt + - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User diff --git a/charts/templates/karmada_cert.yaml b/charts/templates/karmada_cert.yaml index 219773fe4..707ae2467 100644 --- a/charts/templates/karmada_cert.yaml +++ b/charts/templates/karmada_cert.yaml @@ -12,6 +12,12 @@ data: {{ b64enc .Values.certs.custom.crt }} karmada.key: | {{ b64enc .Values.certs.custom.key }} + front-proxy-ca.crt: | + {{ b64enc .Values.certs.custom.frontProxyCaCrt }} + front-proxy-client.crt: | + {{ b64enc .Values.certs.custom.frontProxyCrt }} + front-proxy-client.key: | + {{ b64enc .Values.certs.custom.frontProxyKey }} --- apiVersion: v1 kind: Secret diff --git a/charts/templates/pre-install-job.yaml b/charts/templates/pre-install-job.yaml index a71b01819..7621db73f 100644 --- a/charts/templates/pre-install-job.yaml +++ b/charts/templates/pre-install-job.yaml @@ -24,6 +24,12 @@ data: {{ print "{{ crt }}" }} karmada.key: |- {{ print "{{ key }}" }} + front-proxy-ca.crt: |- + {{ print "{{ front_proxy_ca_crt }}" }} + front-proxy-client.crt: |- + {{ print "{{ front_proxy_crt }}" }} + front-proxy-client.key: |- + {{ print "{{ front_proxy_key }}" }} webhook-cert.yaml: |- apiVersion: v1 kind: Secret @@ -74,6 +80,8 @@ data: {{- include "karmada.webhook.configuration" . | nindent 8 }} {{- print "system-namespace.yaml: " | nindent 6 }} |- {{- include "karmada.systemNamespace" . | nindent 8 }} + {{- print "apiservice.yaml: " | nindent 6 }} |- + {{- include "karmada.apiservice" . | nindent 8 }} crds-configmaps.yaml: |- apiVersion: v1 kind: ConfigMap @@ -150,14 +158,23 @@ spec: mkdir -p /opt/certs cp -r -L /opt/mount/* /opt/configs/ openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "/opt/certs/server-ca.key" -out "/opt/certs/server-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/" + openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "/opt/certs/front-proxy-ca.key" -out "/opt/certs/front-proxy-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/" echo '{"signing":{"default":{"expiry":{{ printf `"%s"` .Values.certs.auto.expiry }},"usages":["signing","key encipherment","client auth","server auth"]}}}' > "/opt/certs/server-ca-config.json" echo '{"CN":"system:admin","hosts":{{ tpl (toJson .Values.certs.auto.hosts) . }},"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=/opt/certs/server-ca.crt -ca-key=/opt/certs/server-ca.key -config=/opt/certs/server-ca-config.json - | cfssljson -bare /opt/certs/karmada + echo '{"signing":{"default":{"expiry":{{ printf `"%s"` .Values.certs.auto.expiry }},"usages":["signing","key encipherment","client auth","server auth"]}}}' > "/opt/certs/front-proxy-ca-config.json" + echo '{"CN":"front-proxy-client","hosts":{{ tpl (toJson .Values.certs.auto.hosts) . }},"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=/opt/certs/front-proxy-ca.crt -ca-key=/opt/certs/front-proxy-ca.key -config=/opt/certs/front-proxy-ca-config.json - | cfssljson -bare /opt/certs/front-proxy-client karmada_ca=$(base64 /opt/certs/server-ca.crt | tr -d '\r\n') karmada_crt=$(base64 /opt/certs/karmada.pem | tr -d '\r\n') karmada_key=$(base64 /opt/certs/karmada-key.pem | tr -d '\r\n') + front_proxy_ca=$(base64 /opt/certs/front-proxy-ca.crt | tr -d '\r\n') + front_proxy_client_crt=$(base64 /opt/certs/front-proxy-client.pem | tr -d '\r\n') + front_proxy_client_key=$(base64 /opt/certs/front-proxy-client-key.pem | tr -d '\r\n') sed -i'' -e "s/{{ print "{{ ca_crt }}" }}/${karmada_ca}/g" /opt/configs/cert.yaml sed -i'' -e "s/{{ print "{{ crt }}" }}/${karmada_crt}/g" /opt/configs/cert.yaml sed -i'' -e "s/{{ print "{{ key }}" }}/${karmada_key}/g" /opt/configs/cert.yaml + sed -i'' -e "s/{{ print "{{ front_proxy_ca_crt }}" }}/${front_proxy_ca}/g" /opt/configs/cert.yaml + sed -i'' -e "s/{{ print "{{ front_proxy_crt }}" }}/${front_proxy_client_crt}/g" /opt/configs/cert.yaml + sed -i'' -e "s/{{ print "{{ front_proxy_key }}" }}/${front_proxy_client_key}/g" /opt/configs/cert.yaml sed -i'' -e "s/{{ print "{{ ca_crt }}" }}/${karmada_ca}/g" /opt/configs/kubeconfig.yaml sed -i'' -e "s/{{ print "{{ crt }}" }}/${karmada_crt}/g" /opt/configs/kubeconfig.yaml sed -i'' -e "s/{{ print "{{ key }}" }}/${karmada_key}/g" /opt/configs/kubeconfig.yaml diff --git a/charts/values.yaml b/charts/values.yaml index 572912538..4c2da7b37 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -38,8 +38,8 @@ certs: ## @param certs.auto.hosts hosts of the certificate hosts: [ "kubernetes.default.svc", - "*.etcd.{{ .Release.Namespace }}.svc.cluster.local", - "*.{{ .Release.Namespace }}.svc.cluster.local", + "*.etcd.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}", + "*.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}", "*.{{ .Release.Namespace }}.svc", "localhost", "127.0.0.1" @@ -60,6 +60,21 @@ certs: -----BEGIN RSA PRIVATE KEY----- XXXXXXXXXXXXXXXXXXXXXXXXXXX -----END RSA PRIVATE KEY----- + ## @param certs.custom.frontProxyCaCrt ca of the front proxy certificate + frontProxyCaCrt: | + -----BEGIN CERTIFICATE----- + XXXXXXXXXXXXXXXXXXXXXXXXXXX + -----END CERTIFICATE----- + ## @param certs.custom.frontProxyCrt crt of the front proxy certificate + frontProxyCrt: | + -----BEGIN CERTIFICATE----- + XXXXXXXXXXXXXXXXXXXXXXXXXXX + -----END CERTIFICATE----- + ## @param certs.custom.frontProxyKey key of the front proxy certificate + frontProxyKey: | + -----BEGIN RSA PRIVATE KEY----- + XXXXXXXXXXXXXXXXXXXXXXXXXXX + -----END RSA PRIVATE KEY----- ## scheduler config scheduler: @@ -230,6 +245,47 @@ apiServer: ## If no port is specified, the nodePort will be automatically assigned. nodePort: 0 +## karmada aggregated apiserver config +aggregatedApiServer: + ## @param aggregatedApiServer.labels + labels: + app: karmada-aggregated-apiserver + ## @param aggregatedApiServer.replicaCount target replicas + replicaCount: 1 + ## @param aggregatedApiServer.podAnnotations + podAnnotations: { } + ## @param aggregatedApiServer.podLabels + podLabels: { } + ## @param aggregatedApiServer.imagePullSecrets + imagePullSecrets: [] + image: + ## @param aggregatedApiServer.image.repository image of the apiserver + repository: swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-aggregated-apiserver + ## @param aggregatedApiServer.image.pullPolicy pull policy of image + pullPolicy: IfNotPresent + ## @param aggregatedApiServer.image.tag overrides the image tag whose default is the latest + tag: latest + ## @param aggregatedApiServer.resources + resources: + requests: + cpu: 100m + # If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + ## @param aggregatedApiServer.nodeSelector + nodeSelector: { } + ## @param aggregatedApiServer.affinity + affinity: { } + ## @param aggregatedApiServer.tolerations + tolerations: [ ] + # - key: node-role.kubernetes.io/master + # operator: Exists + ## kubernetes controller manager config kubeControllerManager: ## @param kubeControllerManager.labels