karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support install karmada aggregated apiserver by helm 

Signed-off-by: jrkeen <jrkeen@hotmail.com>
This commit is contained in:
jrkeen 2021-12-27 23:13:28 +08:00
parent 5136be2917
commit 128fc249e8
10 changed files with 257 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
charts/README.md
View File

@ -112,6 +112,9 @@ $ helm install karmada-scheduler-estimator -n karmada-system ./charts
|`certs.custom.caCrt`|CA CRT of the certificate|`""`|
|`certs.custom.crt`|CRT of the certificate|`""`|
|`certs.custom.key`|KEY of the certificate|`""`|
|`certs.custom.frontProxyCaCrt`|CA CRT of the front proxy certificate|`""`|
|`certs.custom.frontProxyCrt`|CRT of the front proxy certificate|`""`|
|`certs.custom.frontProxyKey`|KEY of the front proxy certificate|`""`|
|`etcd.mode`| Mode "external" and "internal" are provided, "external" means use external ectd, "internal" means install a etcd in the cluster |`"internal"`|
|`etcd.external.servers`| Servers of etcd |`""`|
|`etcd.external.registryPrefix`| Use to registry prefix of etcd |`"/registry/karmada"`|
@ -194,6 +197,18 @@ $ helm install karmada-scheduler-estimator -n karmada-system ./charts
|`apiServer.tolerations`| Tolerations of the karmada-apiserver |`[]`|
|`apiServer.serviceType`| Service type of apiserver, accepts "ClusterIP", "NodePort", "LoadBalancer" |`"ClusterIP"`|
|`apiServer.nodePort`| Node port for apiserver, takes effect when `apiServer.serviceType` is "NodePort". If no port is specified, the nodePort will be automatically assigned. |`0`|
|`aggregatedApiServer.labels`| Labels of the karmada-aggregated-apiserver deployment |`{"app": "karmada-aggregated-apiserver"}`|
|`aggregatedApiServer.replicaCount`| Target replicas of the karmada-aggregated-apiserver |`1`|
|`aggregatedApiServer.podLabels`| Labels of the karmada-aggregated-apiserver pods |`{}`|
|`aggregatedApiServer.podAnnotations`| Annotaions of the karmada-aggregated-apiserver pods |`{}`|
|`aggregatedApiServer.imagePullSecrets`| Image pull secret of the karmada-aggregated-apiserver |`[]`|
|`aggregatedApiServer.image.repository`| Image of the karmada-aggregated-apiserver |`"swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-aggregated-apiserver"`|
|`aggregatedApiServer.image.tag`| Image tag of the karmada-aggregated-apiserver |`"latest"`|
|`aggregatedApiServer.image.pullPolicy`| Image pull policy of the karmada-aggregated-apiserver |`"IfNotPresent"`|
|`aggregatedApiServer.resources`| Resource quota of the karmada-aggregated-apiserver |`{requests: {cpu: 100m}}`|
|`aggregatedApiServer.nodeSelector`| Node selector of the karmada-aggregated-apiserver |`{}`|
|`aggregatedApiServer.affinity`| Affinity of the karmada-aggregated-apiserver |`{}`|
|`aggregatedApiServer.tolerations`| Tolerations of the karmada-aggregated-apiserver |`[]`|
|`kubeControllerManager.labels`| Labels of the kube-controller-manager deployment |`{"app": "kube-controller-manager"}`|
|`kubeControllerManager.replicaCount`| Target replicas of the kube-controller-manager |`1`|
|`kubeControllerManager.podLabels`| Labels of the kube-controller-manager pods |`{}`|

18
charts/templates/_helpers.tpl
View File

@ -26,6 +26,24 @@ app: {{- include "karmada.name" .}}-apiserver
{{- end }}
{{- end -}}
{{- define "karmada.aggregatedApiserver.labels" -}}
{{- if .Values.aggregatedApiServer.labels }}
{{- range $key, $value := .Values.aggregatedApiServer.labels}}
{{ $key }}: {{ $value }}
{{- end}}
{{- else}}
app: {{- include "karmada.name" .}}-aggregated-apiserver
{{- end }}
{{- end -}}
{{- define "karmada.aggregatedApiserver.podLabels" -}}
{{- if .Values.aggregatedApiServer.podLabels }}
{{- range $key, $value := .Values.aggregatedApiServer.podLabels}}
{{ $key }}: {{ $value }}
{{- end}}
{{- end }}
{{- end -}}
{{- define "karmada.kube-cm.labels" -}}
{{- if .Values.kubeControllerManager.labels }}
{{- range $key, $value := .Values.kubeControllerManager.labels}}

30
charts/templates/_karmada_apiservice.tpl Normal file
View File

@ -0,0 +1,30 @@
{{- define "karmada.apiservice" -}}
{{- $name := include "karmada.name" . -}}
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
name: v1alpha1.cluster.karmada.io
labels:
app: {{ $name }}-aggregated-apiserver
apiserver: "true"
spec:
insecureSkipTLSVerify: true
group: cluster.karmada.io
groupPriorityMinimum: 2000
service:
name: {{ $name }}-aggregated-apiserver
namespace: {{ include "karmada.namespace" . }}
version: v1alpha1
versionPriority: 10
---
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}-aggregated-apiserver
namespace: {{ include "karmada.namespace" . }}
spec:
type: ExternalName
externalName: {{ $name }}-aggregated-apiserver.{{ include "karmada.namespace" . }}.svc.{{ .Values.clusterDomain }}
---
{{- end -}}

2
charts/templates/_karmada_system_namespace.tpl
View File

@ -3,7 +3,7 @@
apiVersion: v1
kind: Namespace
metadata:
name: karmada-system
name: {{ include "karmada.namespace" . }}
---
apiVersion: v1
kind: Namespace

2
charts/templates/_karmada_webhook_configuration.tpl
View File

@ -60,7 +60,7 @@ webhooks:
scope: "Cluster"
clientConfig:
url: https://karmada-webhook.karmada-system.svc:443/validate-clusteroverridepolicy
caBundle: {{caBundle}}
{{- include "karmada.webhook.caBundle" . | nindent 6 }}
failurePolicy: Fail
sideEffects: None
admissionReviewVersions: ["v1"]

108
charts/templates/karmada-aggregated-apiserver.yaml Normal file
View File

@ -0,0 +1,108 @@
{{- if eq .Values.installMode "host" }}
{{- $name := include "karmada.name" . -}}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ $name }}-aggregated-apiserver
namespace: {{ include "karmada.namespace" . }}
labels:
{{- include "karmada.aggregatedApiserver.labels" . | nindent 4}}
spec:
selector:
matchLabels:
{{- include "karmada.aggregatedApiserver.labels" . | nindent 6}}
replicas: {{ .Values.aggregatedApiServer.replicaCount }}
template:
metadata:
{{- with .Values.aggregatedApiServer.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "karmada.aggregatedApiserver.labels" . | nindent 8}}
{{- include "karmada.aggregatedApiserver.podLabels" . | nindent 8}}
spec:
automountServiceAccountToken: false
containers:
- name: {{ $name }}-aggregated-apiserver
image: "{{ .Values.aggregatedApiServer.image.repository }}:{{ .Values.aggregatedApiServer.image.tag | default "latest" }}"
imagePullPolicy: {{ .Values.aggregatedApiServer.image.pullPolicy }}
volumeMounts:
{{- include "karmada.kubeconfig.volumeMount" . | nindent 12}}
- name: etcd-cert
mountPath: /etc/etcd/pki
readOnly: true
- name: apiserver-cert
mountPath: /etc/kubernetes/pki
readOnly: true
command:
- /bin/karmada-aggregated-apiserver
- --kubeconfig=/etc/kubeconfig
- --authentication-kubeconfig=/etc/kubeconfig
- --authorization-kubeconfig=/etc/kubeconfig
- --karmada-config=/etc/kubeconfig
{{- if eq .Values.etcd.mode "external" }}
- --etcd-cafile=/etc/etcd/pki/ca.crt
- --etcd-certfile=/etc/etcd/pki/tls.crt
- --etcd-keyfile=/etc/etcd/pki/tls.key
- --etcd-servers={{ .Values.etcd.external.servers }}
- --etcd-prefix={{ .Values.etcd.external.registryPrefix }}
{{- end }}
{{- if eq .Values.etcd.mode "internal" }}
- --etcd-cafile=/etc/etcd/pki/server-ca.crt
- --etcd-certfile=/etc/etcd/pki/karmada.crt
- --etcd-keyfile=/etc/etcd/pki/karmada.key
- --etcd-servers=https://etcd-client.{{ include "karmada.namespace" . }}.svc.{{ .Values.clusterDomain }}:2379
{{- end }}
- --tls-cert-file=/etc/kubernetes/pki/karmada.crt
- --tls-private-key-file=/etc/kubernetes/pki/karmada.key
- --audit-log-path=-
- --feature-gates=APIPriorityAndFairness=false
- --audit-log-maxage=0
- --audit-log-maxbackup=0
resources:
{{- toYaml .Values.aggregatedApiServer.resources | nindent 12 }}
{{- with .Values.aggregatedApiServer.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.aggregatedApiServer.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.aggregatedApiServer.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
volumes:
{{- include "karmada.kubeconfig.volume" . | nindent 8}}
- name: apiserver-cert
secret:
secretName: {{ $name }}-cert
- name: etcd-cert
secret:
{{- if eq .Values.etcd.mode "internal" }}
secretName: {{ $name }}-cert
{{- end }}
{{- if eq .Values.etcd.mode "external" }}
secretName: external-etcd-cert
{{- end }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}-aggregated-apiserver
namespace: {{ include "karmada.namespace" . }}
labels:
{{- include "karmada.aggregatedApiserver.labels" . | nindent 4}}
spec:
ports:
- port: 443
protocol: TCP
targetPort: 443
selector:
{{- include "karmada.aggregatedApiserver.labels" . | nindent 4}}
{{- end}}

6
charts/templates/karmada_apiserver.yaml
View File

@ -61,10 +61,10 @@ spec:
- --service-account-key-file=/etc/kubernetes/pki/karmada.key
- --service-account-signing-key-file=/etc/kubernetes/pki/karmada.key
- --service-cluster-ip-range=10.96.0.0/12
- --proxy-client-cert-file=/etc/kubernetes/pki/karmada.crt
- --proxy-client-key-file=/etc/kubernetes/pki/karmada.key
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --requestheader-allowed-names=front-proxy-client
- --requestheader-client-ca-file=/etc/kubernetes/pki/server-ca.crt
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User

6
charts/templates/karmada_cert.yaml
View File

@ -12,6 +12,12 @@ data:
{{ b64enc .Values.certs.custom.crt }}
karmada.key: |
{{ b64enc .Values.certs.custom.key }}
front-proxy-ca.crt: |
{{ b64enc .Values.certs.custom.frontProxyCaCrt }}
front-proxy-client.crt: |
{{ b64enc .Values.certs.custom.frontProxyCrt }}
front-proxy-client.key: |
{{ b64enc .Values.certs.custom.frontProxyKey }}
---
apiVersion: v1
kind: Secret

17
charts/templates/pre-install-job.yaml
View File

@ -24,6 +24,12 @@ data:
{{ print "{{ crt }}" }}
karmada.key: |-
{{ print "{{ key }}" }}
front-proxy-ca.crt: |-
{{ print "{{ front_proxy_ca_crt }}" }}
front-proxy-client.crt: |-
{{ print "{{ front_proxy_crt }}" }}
front-proxy-client.key: |-
{{ print "{{ front_proxy_key }}" }}
webhook-cert.yaml: |-
apiVersion: v1
kind: Secret
@ -74,6 +80,8 @@ data:
{{- include "karmada.webhook.configuration" . | nindent 8 }}
{{- print "system-namespace.yaml: " | nindent 6 }} |-
{{- include "karmada.systemNamespace" . | nindent 8 }}
{{- print "apiservice.yaml: " | nindent 6 }} |-
{{- include "karmada.apiservice" . | nindent 8 }}
crds-configmaps.yaml: |-
apiVersion: v1
kind: ConfigMap
@ -150,14 +158,23 @@ spec:
mkdir -p /opt/certs
cp -r -L /opt/mount/* /opt/configs/
openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "/opt/certs/server-ca.key" -out "/opt/certs/server-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "/opt/certs/front-proxy-ca.key" -out "/opt/certs/front-proxy-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
echo '{"signing":{"default":{"expiry":{{ printf `"%s"` .Values.certs.auto.expiry }},"usages":["signing","key encipherment","client auth","server auth"]}}}' > "/opt/certs/server-ca-config.json"
echo '{"CN":"system:admin","hosts":{{ tpl (toJson .Values.certs.auto.hosts) . }},"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=/opt/certs/server-ca.crt -ca-key=/opt/certs/server-ca.key -config=/opt/certs/server-ca-config.json - | cfssljson -bare /opt/certs/karmada
echo '{"signing":{"default":{"expiry":{{ printf `"%s"` .Values.certs.auto.expiry }},"usages":["signing","key encipherment","client auth","server auth"]}}}' > "/opt/certs/front-proxy-ca-config.json"
echo '{"CN":"front-proxy-client","hosts":{{ tpl (toJson .Values.certs.auto.hosts) . }},"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=/opt/certs/front-proxy-ca.crt -ca-key=/opt/certs/front-proxy-ca.key -config=/opt/certs/front-proxy-ca-config.json - | cfssljson -bare /opt/certs/front-proxy-client
karmada_ca=$(base64 /opt/certs/server-ca.crt | tr -d '\r\n')
karmada_crt=$(base64 /opt/certs/karmada.pem | tr -d '\r\n')
karmada_key=$(base64 /opt/certs/karmada-key.pem | tr -d '\r\n')
front_proxy_ca=$(base64 /opt/certs/front-proxy-ca.crt | tr -d '\r\n')
front_proxy_client_crt=$(base64 /opt/certs/front-proxy-client.pem | tr -d '\r\n')
front_proxy_client_key=$(base64 /opt/certs/front-proxy-client-key.pem | tr -d '\r\n')
sed -i'' -e "s/{{ print "{{ ca_crt }}" }}/${karmada_ca}/g" /opt/configs/cert.yaml
sed -i'' -e "s/{{ print "{{ crt }}" }}/${karmada_crt}/g" /opt/configs/cert.yaml
sed -i'' -e "s/{{ print "{{ key }}" }}/${karmada_key}/g" /opt/configs/cert.yaml
sed -i'' -e "s/{{ print "{{ front_proxy_ca_crt }}" }}/${front_proxy_ca}/g" /opt/configs/cert.yaml
sed -i'' -e "s/{{ print "{{ front_proxy_crt }}" }}/${front_proxy_client_crt}/g" /opt/configs/cert.yaml
sed -i'' -e "s/{{ print "{{ front_proxy_key }}" }}/${front_proxy_client_key}/g" /opt/configs/cert.yaml
sed -i'' -e "s/{{ print "{{ ca_crt }}" }}/${karmada_ca}/g" /opt/configs/kubeconfig.yaml
sed -i'' -e "s/{{ print "{{ crt }}" }}/${karmada_crt}/g" /opt/configs/kubeconfig.yaml
sed -i'' -e "s/{{ print "{{ key }}" }}/${karmada_key}/g" /opt/configs/kubeconfig.yaml

60
charts/values.yaml
View File

@ -38,8 +38,8 @@ certs:
## @param certs.auto.hosts hosts of the certificate
hosts: [
"kubernetes.default.svc",
"*.etcd.{{ .Release.Namespace }}.svc.cluster.local",
"*.{{ .Release.Namespace }}.svc.cluster.local",
"*.etcd.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}",
"*.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}",
"*.{{ .Release.Namespace }}.svc",
"localhost",
"127.0.0.1"
@ -60,6 +60,21 @@ certs:
-----BEGIN RSA PRIVATE KEY-----
XXXXXXXXXXXXXXXXXXXXXXXXXXX
-----END RSA PRIVATE KEY-----
## @param certs.custom.frontProxyCaCrt ca of the front proxy certificate
frontProxyCaCrt: |
-----BEGIN CERTIFICATE-----
XXXXXXXXXXXXXXXXXXXXXXXXXXX
-----END CERTIFICATE-----
## @param certs.custom.frontProxyCrt crt of the front proxy certificate
frontProxyCrt: |
-----BEGIN CERTIFICATE-----
XXXXXXXXXXXXXXXXXXXXXXXXXXX
-----END CERTIFICATE-----
## @param certs.custom.frontProxyKey key of the front proxy certificate
frontProxyKey: |
-----BEGIN RSA PRIVATE KEY-----
XXXXXXXXXXXXXXXXXXXXXXXXXXX
-----END RSA PRIVATE KEY-----
## scheduler config
scheduler:
@ -230,6 +245,47 @@ apiServer:
## If no port is specified, the nodePort will be automatically assigned.
nodePort: 0
## karmada aggregated apiserver config
aggregatedApiServer:
## @param aggregatedApiServer.labels
labels:
app: karmada-aggregated-apiserver
## @param aggregatedApiServer.replicaCount target replicas
replicaCount: 1
## @param aggregatedApiServer.podAnnotations
podAnnotations: { }
## @param aggregatedApiServer.podLabels
podLabels: { }
## @param aggregatedApiServer.imagePullSecrets
imagePullSecrets: []
image:
## @param aggregatedApiServer.image.repository image of the apiserver
repository: swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-aggregated-apiserver
## @param aggregatedApiServer.image.pullPolicy pull policy of image
pullPolicy: IfNotPresent
## @param aggregatedApiServer.image.tag overrides the image tag whose default is the latest
tag: latest
## @param aggregatedApiServer.resources
resources:
requests:
cpu: 100m
# If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
## @param aggregatedApiServer.nodeSelector
nodeSelector: { }
## @param aggregatedApiServer.affinity
affinity: { }
## @param aggregatedApiServer.tolerations
tolerations: [ ]
# - key: node-role.kubernetes.io/master
# operator: Exists
## kubernetes controller manager config
kubeControllerManager:
## @param kubeControllerManager.labels