From 6a2e5ed52fa573c0d7b4993f6de71ebf52a620b8 Mon Sep 17 00:00:00 2001 From: jackzhang Date: Mon, 6 Dec 2021 15:15:20 +0800 Subject: [PATCH] Enhance helm chart to support more scenes Signed-off-by: jackzhang --- charts/Chart.yaml | 2 +- charts/README.md | 7 +++++ charts/templates/etcd.yaml | 35 +++++++++++++++++++++-- charts/templates/karmada_apiserver.yaml | 9 +++++- charts/templates/post-install-job.yaml | 1 + charts/templates/pre-install-job.yaml | 4 ++- charts/values.yaml | 38 +++++++++++++++++++++++-- 7 files changed, 87 insertions(+), 9 deletions(-) diff --git a/charts/Chart.yaml b/charts/Chart.yaml index bf3d1e04e..77622df5f 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -25,4 +25,4 @@ version: 0.0.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: v0.8.0 +appVersion: v0.10.1 diff --git a/charts/README.md b/charts/README.md index 250cc5c55..237036ac3 100644 --- a/charts/README.md +++ b/charts/README.md @@ -122,6 +122,10 @@ $ helm install karmada-scheduler-estimator -n karmada-system ./charts |`etcd.internal.image.repository`| Image of the etcd |`"k8s.gcr.io/etcd"`| |`etcd.internal.image.pullPolicy`| Image pull policy of the etcd |`"IfNotPresent"`| |`etcd.internal.image.tag`| Image tag of the etcd |`"3.4.13-0"`| +|`etcd.internal.storageType`| StorageType of the etcd, accepts "hostPath", "pvc" |`"hostPath"`| +|`etcd.internal.pvc.storageClass`| StorageClass of the etcd, takes effect when `etcd.internal.storageType` is "pvc" |`""`| +|`etcd.internal.pvc.size`| Storage size of the etcd, takes effect when `etcd.internal.storageType` is "pvc" |`""`| +|`etcd.internal.resources`| Resource quota of the etcd |`{}`| |`agent.clusterName`| Name of the member cluster |`""`| |`agent.kubeconfig.caCrt`| CA CRT of the karmada certificate |`""`| |`agent.kubeconfig.crt`| CRT of the karmada certificate |`""`| @@ -184,9 +188,12 @@ $ helm install karmada-scheduler-estimator -n karmada-system ./charts |`apiServer.image.tag`| Image tag of the karmada-apiserver |`"v1.19.1"`| |`apiServer.image.pullPolicy`| Image pull policy of the karmada-apiserver |`"IfNotPresent"`| |`apiServer.resources`| Resource quota of the karmada-apiserver |`{}`| +|`apiServer.hostNetwork`| Deploy karmada-apiserver with hostNetwork. If there are multiple karmadas in one cluster, you'd better set it to "false" |`"true"`| |`apiServer.nodeSelector`| Node selector of the karmada-apiserver |`{}`| |`apiServer.affinity`| Affinity of the karmada-apiserver |`{}`| |`apiServer.tolerations`| Tolerations of the karmada-apiserver |`[]`| +|`apiServer.serviceType`| Service type of apiserver, accepts "ClusterIP", "NodePort", "LoadBalancer" |`"ClusterIP"`| +|`apiServer.nodePort`| Node port for apiserver, takes effect when `apiServer.serviceType` is "NodePort". If no port is specified, the nodePort will be automatically assigned. |`0`| |`kubeControllerManager.labels`| Labels of the kube-controller-manager deployment |`{"app": "kube-controller-manager"}`| |`kubeControllerManager.replicaCount`| Target replicas of the kube-controller-manager |`1`| |`kubeControllerManager.podLabels`| Labels of the kube-controller-manager pods |`{}`| diff --git a/charts/templates/etcd.yaml b/charts/templates/etcd.yaml index 5f6092657..e79c0a3bd 100644 --- a/charts/templates/etcd.yaml +++ b/charts/templates/etcd.yaml @@ -1,3 +1,12 @@ +{{- define "etcd.initial.clusters" -}} + {{- $uri := "" -}} + {{- $ns := include "karmada.namespace" . -}} + {{- range $index := until (.Values.etcd.internal.replicaCount | int) -}} + {{- $node := printf "etcd-%d=http://etcd-%d.etcd.%s.svc.%s:2380," $index $index $ns $.Values.clusterDomain }} + {{- $uri = printf "%s%s" $uri $node -}} + {{- end -}} + {{- printf $uri | trimSuffix "," | quote -}} +{{- end }} {{- if and (eq .Values.etcd.mode "internal") (eq .Values.installMode "host")}} apiVersion: apps/v1 kind: StatefulSet @@ -46,6 +55,11 @@ spec: periodSeconds: 60 successThreshold: 1 timeoutSeconds: 10 + env: + - name: KARMADA_ETCD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - containerPort: 2369 name: client @@ -53,6 +67,8 @@ spec: - containerPort: 2370 name: server protocol: TCP + resources: + {{- toYaml .Values.etcd.internal.resources | nindent 12 }} volumeMounts: - mountPath: /var/lib/etcd name: etcd-data @@ -62,7 +78,7 @@ spec: command: - /usr/local/bin/etcd - --name - - etcd0 + - $(KARMADA_ETCD_NAME) - --listen-peer-urls - http://0.0.0.0:2380 - --listen-client-urls @@ -70,7 +86,7 @@ spec: - --advertise-client-urls - https://etcd-client.{{ include "karmada.namespace" . }}.svc.{{ .Values.clusterDomain }}:2379 - --initial-cluster - - etcd0=http://etcd-0.etcd.{{ include "karmada.namespace" . }}.svc.{{ .Values.clusterDomain }}:2380 + - {{ include "etcd.initial.clusters" . }} - --initial-cluster-state - new - --cert-file=/etc/kubernetes/pki/etcd/karmada.crt @@ -82,10 +98,23 @@ spec: - name: etcd-cert secret: secretName: {{ include "karmada.name" . }}-cert + {{- if eq .Values.etcd.internal.storageType "hostPath" }} - hostPath: - path: /var/lib/karmada-etcd + path: /var/lib/{{ include "karmada.namespace" . }}/karmada-etcd type: DirectoryOrCreate name: etcd-data + {{- end}} + {{- if eq .Values.etcd.internal.storageType "pvc"}} + volumeClaimTemplates: + - metadata: + name: etcd-data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ .Values.etcd.internal.pvc.storageClass }} + resources: + requests: + storage: {{ .Values.etcd.internal.pvc.size }} + {{- end}} --- apiVersion: v1 kind: Service diff --git a/charts/templates/karmada_apiserver.yaml b/charts/templates/karmada_apiserver.yaml index 366ad3899..e89a36141 100644 --- a/charts/templates/karmada_apiserver.yaml +++ b/charts/templates/karmada_apiserver.yaml @@ -104,9 +104,13 @@ spec: - name: etcd-cert mountPath: /etc/etcd/pki readOnly: true + {{- if .Values.apiServer.hostNetwork }} dnsPolicy: ClusterFirstWithHostNet + {{- end }} enableServiceLinks: true + {{- if .Values.apiServer.hostNetwork }} hostNetwork: true + {{- end }} preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical @@ -146,9 +150,12 @@ metadata: labels: {{- include "karmada.apiserver.labels" . | nindent 4}} spec: - type: ClusterIP + type: {{ .Values.apiServer.serviceType }} ports: - name: {{ $name }}-apiserver + {{- if and (eq .Values.apiServer.serviceType "NodePort") (.Values.apiServer.nodePort) }} + nodePort: {{ .Values.apiServer.nodePort }} + {{- end }} port: 5443 protocol: TCP targetPort: 5443 diff --git a/charts/templates/post-install-job.yaml b/charts/templates/post-install-job.yaml index 2fe085589..36be1c229 100644 --- a/charts/templates/post-install-job.yaml +++ b/charts/templates/post-install-job.yaml @@ -80,6 +80,7 @@ spec: containers: - name: post-install image: {{ .Values.postInstallJob.postInstallContainerImage }} + imagePullPolicy: IfNotPresent command: - /bin/sh - -c diff --git a/charts/templates/pre-install-job.yaml b/charts/templates/pre-install-job.yaml index 4e54fd75f..a71b01819 100644 --- a/charts/templates/pre-install-job.yaml +++ b/charts/templates/pre-install-job.yaml @@ -138,6 +138,7 @@ spec: initContainers: - name: init image: {{ .Values.preInstallJob.initContainerImage }} + imagePullPolicy: IfNotPresent workingDir: /opt/mount command: - /bin/sh @@ -150,7 +151,7 @@ spec: cp -r -L /opt/mount/* /opt/configs/ openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "/opt/certs/server-ca.key" -out "/opt/certs/server-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/" echo '{"signing":{"default":{"expiry":{{ printf `"%s"` .Values.certs.auto.expiry }},"usages":["signing","key encipherment","client auth","server auth"]}}}' > "/opt/certs/server-ca-config.json" - echo '{"CN":"system:admin","hosts":{{ toJson .Values.certs.auto.hosts }},"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=/opt/certs/server-ca.crt -ca-key=/opt/certs/server-ca.key -config=/opt/certs/server-ca-config.json - | cfssljson -bare /opt/certs/karmada + echo '{"CN":"system:admin","hosts":{{ tpl (toJson .Values.certs.auto.hosts) . }},"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=/opt/certs/server-ca.crt -ca-key=/opt/certs/server-ca.key -config=/opt/certs/server-ca-config.json - | cfssljson -bare /opt/certs/karmada karmada_ca=$(base64 /opt/certs/server-ca.crt | tr -d '\r\n') karmada_crt=$(base64 /opt/certs/karmada.pem | tr -d '\r\n') karmada_key=$(base64 /opt/certs/karmada-key.pem | tr -d '\r\n') @@ -173,6 +174,7 @@ spec: containers: - name: pre-install image: {{ .Values.preInstallJob.preInstallContainerImage }} + imagePullPolicy: IfNotPresent workingDir: /opt/mount command: - /bin/sh diff --git a/charts/values.yaml b/charts/values.yaml index 80db79530..74c8ad5a2 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -38,9 +38,9 @@ certs: ## @param certs.auto.hosts hosts of the certificate hosts: [ "kubernetes.default.svc", - "*.etcd.karmada-system.svc.cluster.local", - "*.karmada-system.svc.cluster.local", - "*.karmada-system.svc", + "*.etcd.{{ .Release.Namespace }}.svc.cluster.local", + "*.{{ .Release.Namespace }}.svc.cluster.local", + "*.{{ .Release.Namespace }}.svc", "localhost", "127.0.0.1" ] @@ -208,6 +208,10 @@ apiServer: # requests: # cpu: 100m # memory: 128Mi + ## @param apiServer.hostNetwork + ## "true" means using hostNetwork + ## "false" means normal network + hostNetwork: true ## @param apiServer.nodeSelector nodeSelector: { } ## @param apiServer.affinity @@ -216,6 +220,15 @@ apiServer: tolerations: [ ] # - key: node-role.kubernetes.io/master # operator: Exists + ## @param apiServer.serviceType default service type for apiserver + ## "LoadBalancer" means using LoadBalancer + ## "ClusterIP" means using ClusterIP + ## "NodePort" means using NodePort + serviceType: ClusterIP + ## @param apiServer.nodePort node port for apiserver service, + ## will take effect when 'apiServer.serviceType' is 'NodePort'. + ## If no port is specified, the nodePort will be automatically assigned. + nodePort: 0 ## kubernetes controller manager config kubeControllerManager: @@ -294,6 +307,25 @@ etcd: pullPolicy: IfNotPresent ## @param etcd.internal.image.tag overrides the image tag whose default is the latest tag: "3.4.13-0" + ## @param etcd.internal.storageType storage type for etcd data + ## "pvc" means using volumeClaimTemplates + ## "hostPath" means using hostPath + storageType: "hostPath" + pvc: + ## @param etcd.internal.pvc.storageClass storageClass name of PVC + storageClass: "" + ## @param etcd.internal.pvc.size size of PVC + size: "" + ## @param etcd.internal.resources + resources: { } + # If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi ## agent client config agent: