deploy karmada etcd and karmada apiserver

2020-12-15 20:16:44 +08:00 · 2020-12-15 20:16:44 +08:00 · 786538765a
parent a35b577afe
commit 786538765a
10 changed files with 630 additions and 22 deletions
--- a/README.md
+++ b/README.md
@ -94,12 +94,14 @@ hack/create-cluster.sh member-cluster-1 /root/.kube/membercluster1.config

 ```
 make karmadactl
+export KUBECONFIG=/var/run/karmada/karmada-apiserver.config
 ./karmadactl join member-cluster-1 --member-cluster-kubeconfig=/root/.kube/membercluster1.config
 ```

 3. Verify member cluster is Joined to karmada successfully.

 ```
+export KUBECONFIG=/var/run/karmada/karmada-apiserver.config
 kubectl get membercluster -n karmada-cluster
 ```

@ -108,14 +110,14 @@ kubectl get membercluster -n karmada-cluster
 1. Create nginx deployment in karmada.

 ```
-export KUBECONFIG=/root/.kube/karmada.config
+export KUBECONFIG=/var/run/karmada/karmada-apiserver.config
 kubectl create -f samples/nginx/deployment.yaml
 ```

 2. Create PropagationPolicy that will propagate nginx to member cluster.

 ```
-export KUBECONFIG=/root/.kube/karmada.config
+export KUBECONFIG=/var/run/karmada/karmada-apiserver.config
 kubectl create -f samples/nginx/propagationpolicy.yaml
 ```

--- a/artifacts/deploy/karmada-apiserver.yaml
+++ b/artifacts/deploy/karmada-apiserver.yaml
@ -0,0 +1,128 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: karmada-apiserver
+  namespace: karmada-system
+  labels:
+    app: karmada-apiserver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: karmada-apiserver
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: karmada-apiserver
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - karmada-apiserver
+              topologyKey: kubernetes.io/hostname
+      containers:
+        - command:
+            - kube-apiserver
+            - --allow-privileged=true
+            - --authorization-mode=Node,RBAC
+            - --client-ca-file=/etc/kubernetes/pki/server-ca.crt
+            - --enable-admission-plugins=NodeRestriction
+            - --enable-bootstrap-token-auth=true
+            - --etcd-cafile=/etc/kubernetes/pki/server-ca.crt
+            - --etcd-certfile=/etc/kubernetes/pki/karmada.crt
+            - --etcd-keyfile=/etc/kubernetes/pki/karmada.key
+            - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379
+            - --insecure-port=8080
+            - --kubelet-client-certificate=/etc/kubernetes/pki/karmada.crt
+            - --kubelet-client-key=/etc/kubernetes/pki/karmada.key
+            - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+            - --runtime-config=
+            - --secure-port=5443
+            - --service-cluster-ip-range=10.96.0.0/12
+            - --proxy-client-cert-file=/etc/kubernetes/pki/karmada.crt
+            - --proxy-client-key-file=/etc/kubernetes/pki/karmada.key
+            - --requestheader-allowed-names=front-proxy-client
+            - --requestheader-client-ca-file=/etc/kubernetes/pki/server-ca.crt
+            - --requestheader-extra-headers-prefix=X-Remote-Extra-
+            - --requestheader-group-headers=X-Remote-Group
+            - --requestheader-username-headers=X-Remote-User
+            - --tls-cert-file=/etc/kubernetes/pki/karmada.crt
+            - --tls-private-key-file=/etc/kubernetes/pki/karmada.key
+          name: karmada-apiserver
+          image: k8s.gcr.io/kube-apiserver:v1.19.1
+          imagePullPolicy: Always
+          livenessProbe:
+            failureThreshold: 8
+            httpGet:
+              host: {{api_addr}}
+              path: /livez
+              port: 5443
+              scheme: HTTPS
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 15
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              host: {{api_addr}}
+              path: /readyz
+              port: 5443
+              scheme: HTTPS
+            periodSeconds: 1
+            successThreshold: 1
+            timeoutSeconds: 15
+          resources:
+            requests:
+              cpu: 250m
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /etc/kubernetes/pki
+              name: k8s-certs
+              readOnly: true
+      dnsPolicy: ClusterFirstWithHostNet
+      enableServiceLinks: true
+      hostNetwork: true
+      preemptionPolicy: PreemptLowerPriority
+      priority: 2000001000
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      tolerations:
+        - effect: NoExecute
+          operator: Exists
+      volumes:
+        - name: k8s-certs
+          secret:
+            secretName: karmada-cert-secret
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: karmad-apiserver
+  namespace: karmada-system
+  labels:
+    app: karmada-apiserver
+spec:
+  ports:
+    - name: karmad-apiserver-kubectl
+      port: 5443
+      protocol: TCP
+      targetPort: 5443
+  selector:
+    app: karmada-apiserver
--- a/artifacts/deploy/karmada-cert-secret.yaml
+++ b/artifacts/deploy/karmada-cert-secret.yaml
@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: karmada-cert-secret
+  namespace: karmada-system
+type: Opaque
+data:
+  server-ca.crt: |
+    {{ca_crt}}
+  karmada.crt: |
+    {{client_cer}}
+  karmada.key: |
+    {{client_key}}
+
--- a/artifacts/deploy/karmada-etcd.yaml
+++ b/artifacts/deploy/karmada-etcd.yaml
@ -0,0 +1,127 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: etcd
+  namespace: karmada-system
+  labels:
+    app: etcd
+spec:
+  replicas: 1
+  serviceName: etcd
+  selector:
+    matchLabels:
+      app: etcd
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: etcd
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - etcd
+              topologyKey: kubernetes.io/hostname
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: etcd
+          image: k8s.gcr.io/etcd:3.4.13-0
+          imagePullPolicy: Always
+          livenessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -ec
+                - 'etcdctl get /registry --prefix --keys-only  --endpoints https://127.0.0.1:2379  --cacert /etc/kubernetes/pki/etcd/server-ca.crt --cert /etc/kubernetes/pki/etcd/karmada.crt --key /etc/kubernetes/pki/etcd/karmada.key'
+            failureThreshold: 3
+            initialDelaySeconds: 600
+            periodSeconds: 60
+            successThreshold: 1
+            timeoutSeconds: 10
+          ports:
+            - containerPort: 2369
+              name: client
+              protocol: TCP
+            - containerPort: 2370
+              name: server
+              protocol: TCP
+          volumeMounts:
+            - mountPath: /var/lib/etcd
+              name: etcd-data
+            - mountPath: /etc/kubernetes/pki/etcd
+              name: etcd-certs
+          command:
+            - /usr/local/bin/etcd
+            - --name
+            - etcd0
+            - --listen-peer-urls
+            - http://0.0.0.0:2380
+            - --listen-client-urls
+            - https://0.0.0.0:2379
+            - --advertise-client-urls
+            - https://etcd-client.karmada-system.svc.cluster.local:2379
+            - --initial-cluster
+            - etcd0=http://etcd-0.etcd.karmada-system.svc.cluster.local:2380
+            - --initial-cluster-state
+            - new
+            - --cert-file=/etc/kubernetes/pki/etcd/karmada.crt
+            - --client-cert-auth=true
+            - --key-file=/etc/kubernetes/pki/etcd/karmada.key
+            - --trusted-ca-file=/etc/kubernetes/pki/etcd/server-ca.crt
+            - --data-dir=/var/lib/etcd
+      volumes:
+        - hostPath:
+            path: /var/lib/karmada-etcd
+            type: DirectoryOrCreate
+          name: etcd-data
+        - name: etcd-certs
+          secret:
+            secretName: karmada-cert-secret
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: etcd
+  name: etcd-client
+  namespace: karmada-system
+spec:
+  ports:
+    - name: etcd-client-port
+      port: 2379
+      protocol: TCP
+      targetPort: 2379
+  selector:
+    app: etcd
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: etcd
+  name: etcd
+  namespace: karmada-system
+spec:
+  ports:
+    - name: client
+      port: 2379
+      protocol: TCP
+      targetPort: 2379
+    - name: server
+      port: 2380
+      protocol: TCP
+      targetPort: 2380
+  clusterIP: None
+  selector:
+    app: etcd
+
--- a/artifacts/deploy/kube-controller-manager.yaml
+++ b/artifacts/deploy/kube-controller-manager.yaml
@ -0,0 +1,94 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: karmada-kube-controller-manager
+  namespace: karmada-system
+  labels:
+    app: kube-controller-manager
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kube-controller-manager
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: kube-controller-manager
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - kube-controller-manager
+              topologyKey: kubernetes.io/hostname
+      containers:
+        - command:
+            - kube-controller-manager
+            - --allocate-node-cidrs=true
+            - --authentication-kubeconfig=/etc/kubeconfig
+            - --authorization-kubeconfig=/etc/kubeconfig
+            - --bind-address=127.0.0.1
+            - --client-ca-file=/etc/karmada/pki/server-ca.crt
+            - --cluster-cidr=10.244.0.0/16
+            - --cluster-name=karmada
+            - --cluster-signing-cert-file=/etc/karmada/pki/server-ca.crt
+            - --cluster-signing-key-file=/etc/karmada/pki/server-ca.key
+            - --controllers=namespace,garbagecollector,serviceaccount-token
+            - --kubeconfig=/etc/kubeconfig
+            - --leader-elect=true
+            - --node-cidr-mask-size=24
+            - --port=0
+            - --root-ca-file=/etc/karmada/pki/server-ca.crt
+            - --service-account-private-key-file=/etc/karmada/pki/karmada.key
+            - --service-cluster-ip-range=10.96.0.0/12
+            - --use-service-account-credentials=true
+          image: k8s.gcr.io/kube-controller-manager:v1.19.1
+          imagePullPolicy: IfNotPresent
+          livenessProbe:
+            failureThreshold: 8
+            httpGet:
+              host: 127.0.0.1
+              path: /healthz
+              port: 10257
+              scheme: HTTPS
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 15
+          name: kube-controller-manager
+          resources:
+            requests:
+              cpu: 200m
+          startupProbe:
+            failureThreshold: 24
+            httpGet:
+              host: 127.0.0.1
+              path: /healthz
+              port: 10257
+              scheme: HTTPS
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 15
+          volumeMounts:
+            - mountPath: /etc/karmada/pki
+              name: k8s-certs
+              readOnly: true
+            - mountPath: /etc/kubeconfig
+              subPath: kubeconfig
+              name: kubeconfig
+      priorityClassName: system-node-critical
+      volumes:
+        - name: k8s-certs
+          secret:
+            secretName: karmada-cert-secret
+        - name: kubeconfig
+          secret:
+            secretName: kubeconfig
--- a/artifacts/deploy/secret.yaml
+++ b/artifacts/deploy/secret.yaml
@ -1,13 +1,11 @@
-# this is a example yaml.
-# You need to replace value of kubeconfig with your own kubeconfig.
 apiVersion: v1
 stringData:
  kubeconfig: |-
    apiVersion: v1
    clusters:
    - cluster:
-        certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJd01USXhNVEEzTkRreU5Gb1hEVE13TVRJd09UQTNORGt5TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTGRZCnRSRGJkMEV1d3lZV3FmYy9jbDZLM0RqMi8vVmFnS1BlSUEwRWNaWENkemxybGFDWlNGNUFvVHE4TWpWbXlZRWYKc3VreG9TbTRJZEVMZDNYUzRwaHBIWGVRUllsMUxTbFJFRXNRcDBLbG1iOGhGa2JmVGppZjdwUUFyYTN2VlV1bAp1Uld3M3dJV0hROTA0WmxpZExoQ2hyYTZzb2pzM20xMXRDT2xGOEpIZTBFSjZkeWxqcjR5K1RORUVOQnJza0w2Cnd5UmlDTHJHbVo3Y2VxbjZiOWp1WFk0cjNiUGNvS3V1UWw4UHltTkQ4SmhBNEZYbldvblJ3ZERNRk9LTjlxQmoKL2dUejREVXMreFMyWEsxUlV1RVFCOGZ4blBtS29Ga0Q4QXBxTHFxcExWTUJjenVtMkdzSk9QYmlZT3FMTmd1awo4Ny9GUFRsT0RRSDhyVjdreWcwQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZIdjBReHZZaWc0ZWZoNk1kalJjZDlJQUJsemFNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDUHl3SmluMkJhc3hOdWp0WlZTaUd5WjIwSkVQVkI3MVc4cVYyS2ZoWHcvWnZpNGpDSApjWlBEYmxzdThqTFhraUgrY3ltWHpjc29CLzhkbWRUbWxuMmVhbjIzaHJhWUpVU1JqTDRNdlhGcTFNNmdxL1UwCkcvenhnaGw5WW5wYlVieGk0UFRuNVhaSFl5VDJ4cUEybW9JdkdmYytoV0NZMVpKMXRwaVRjZUFKY0d0QWxGdjkKdExEVHRiM0FhWW5WTVJkWFMxeDdRLzllV1VoeUFGV2JnQ0hrYjB0YXpQbTM0RGpuYXFSbU01dlV2VkkvTGJFYgpZN28vQ09reHJYekl3KzhGbTB4UXl0NUQ0Z3NrQThkNHVqQ1ZaVCtRT0ltdzRWeGtlQjdOZ3puN043MWNqQzZSCnQwZ2R5RUg1VU9STXBvT0d3VVVCTmpSdGd6cmUyWkYwUVlhOQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
-        server: https://172.17.0.2:6443
+        certificate-authority-data: {{ca_crt}}
+        server: https://karmad-apiserver.karmada-system.svc.cluster.local:5443
      name: kind-karmada
    contexts:
    - context:
@ -20,8 +18,8 @@ stringData:
    users:
    - name: kind-karmada
      user:
-        client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lJQjRsVU9EaDVxa1V3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TURFeU1URXdOelE1TWpSYUZ3MHlNVEV5TVRFd056UTVNalphTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXFZd2o5N0QyZEp0UkhOM1MKUmdtODJyZ0hORE5IYkRHQndUV0Vxck9lQlFyS1JjUVBGWE5MbGJ1cXhvRlc0STdKNk8wd1RaRHBvNUxTNEZSOQptbmZLN2JXM040eno3aWpZRWNYeFFQWStWRFJ0TU9CWVJrdjhIenE3MjgvMXRMQTA5Z1RrK0Z0dFE5ckNjNEFTCnAxSjRNUEhEWHVXVC9TaDdTVkMwTHpPa0IrcWRoRVU5MzQrdkpmc1JCakhXd1IrNW1kaUxQWkFaNHRJZll0SHcKMmlNMkNmV09TeEVINTBackc3aW04aHB5czNNWWdwTE8yajc0K2FhZFZxZFI2QjlNWXJKMC8wbW4zTllLMU5rWgozbFVUdmRyd1NYUmV4YWZFUUVUeHZhekh4dE1aS2w4SlRrRHZqUjhSc1pvMUs5V0hvdkdsUEt1RnVqck41bHUvClFtVm83UUlEQVFBQm8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0h3WURWUjBqQkJnd0ZvQVVlL1JERzlpS0RoNStIb3gyTkZ4MzBnQUdYTm93RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFCN0VrdTM5TzBXN2RWRXdqc3dGYzdYNEtIWWZLQ3g4VCtCaGlHclVCcXhIcE40ckdCOFVoT0JLCnp3SWY1c2E2ZmhJUE82b2xXMlMwalFId1ZCZ1RpbTVhS25uaktObGIySHdLakNYUVBPZ0N2R0Z4c2M0WkxYVFEKVW5JQjcwVWZCUHJOakNFRFBsN3N1UXE0MWIycCtkVzRtV0dYQUNoYXdjWkJZbUtlZTVlNFBkTHdteW4zeWgxMgp6eEFpL1UxUm15c0FFd1ZqV25OMGhtQ3NTcWR0UTQ2eDNpT0gxR1MrbzB4MkUxM2RVMURZQk53d0pZTUh3WnVUCm5YSDRadXl1bjVESnRrNFlENDFFbm1NdFZ5OFhyTEVaR21LbDVjb1JMeWR6Sk4zSXRvY0Q2ZTdIYUJmcTZZRXUKZ0NiNG1RMWhhT2k3UVR0ZWpYTmFVekI5SER6aGdaTT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
-        client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBcVl3ajk3RDJkSnRSSE4zU1JnbTgycmdITkROSGJER0J3VFdFcXJPZUJRcktSY1FQCkZYTkxsYnVxeG9GVzRJN0o2TzB3VFpEcG81TFM0RlI5bW5mSzdiVzNONHp6N2lqWUVjWHhRUFkrVkRSdE1PQlkKUmt2OEh6cTcyOC8xdExBMDlnVGsrRnR0UTlyQ2M0QVNwMUo0TVBIRFh1V1QvU2g3U1ZDMEx6T2tCK3FkaEVVOQozNCt2SmZzUkJqSFd3Uis1bWRpTFBaQVo0dElmWXRIdzJpTTJDZldPU3hFSDUwWnJHN2ltOGhweXMzTVlncExPCjJqNzQrYWFkVnFkUjZCOU1ZckowLzBtbjNOWUsxTmtaM2xVVHZkcndTWFJleGFmRVFFVHh2YXpIeHRNWktsOEoKVGtEdmpSOFJzWm8xSzlXSG92R2xQS3VGdWpyTjVsdS9RbVZvN1FJREFRQUJBb0lCQUYrMU56eUVEYXVuWFhOaApHR2ptNkUvRElIWHNHSDVQdUFKYjlxYnh1OXIyeTFYMHBkc2krV0R0emlvK0tzK2Fhd25za3A3R2xjejdmY1NhCmpVaWNKTlpwQktMOEUxcTJmYm9TdGV4STFNaHR5OUdTWXVKVDFGL0FwUGtoZkg3aUZSTm5rZVZNbnZKMTA5eWwKU0dKaG9HMW9uRE8rZnZxaVZtMzllNmxGaW12WXJwakQvNTFvWW1pd0JoRmpjYzcyMGlmdklmYURhNjBFd0wwcgpCaGViUG1mSFFZRzM3ZEZJeDN5dm4xWndXRVFma2hRcGYzeTlwNUNzOTJEYVBVVDhkR3BkZ3lNSDRTOWVMNlJrClBzbjdpcjNCNWtSZjJsSlVnMnV4eG5jbWVzQSsyZ0VYa3F2aldrc2d5Tk5MdGJ2S01La0dPTW5VU1h3MFBBc3kKa29wN0I3VUNnWUVBMGNMQ1BkVExVWjNBWFBrOWVsQkNUYnhnTndwcTlBSUJxQnVHQWhWSlpwaXpUR1NCeGIrcwo4bmtuZEhGd2d6cVcvdXp6RFZoaDl1MFpXQjF2eGdQdlJpN3B4Z3NIUG1qZjVPZko4KzdCQnhxQU1YdjZyTC9wCjNMYytpZVVYbTI1b1hjZjJtL3VOTitUVFBnOVVDWjlJOXhvUkxQTmQ4TlFabmxDUVd6aDlHaGNDZ1lFQXp1d08KeXR3MEQxdXBscml1Vm5XVWRTT3cwc1F3aEJiRnVsMzE5V2hibEFrS2pydGdiMXFMMXpnam1iZjlnR3RHK0FFWApvWVdsd2YxYUp5K0lKZWV0VndxUUZ1RXg4cmFuQ2xHWGNvSm15QmFFWlZBdFB6aUZWaHN0aXhHUHFLWUw3YUUrClBRTXFRTlZROUdLUExta093L1JpYnRWZUJ5ZjdISExoSzF1WmE1c0NnWUVBejFaL3diWnhNcitId1Y3c0VkYjcKY3ZOYzk0cm9wVURHZW5DYVNiell6UHpyWGZ1Syt2aTM3d1VxcHNMcXdBeE43TFl5bVdKZmswQlI2UE5QNUo5SApDRXllN3c4L25jVDBhc05pc0NlenpWMjRrVEZIV3pKbjY2K0Z0YkFwWVk0RXc0NUFpWFpnNUhyZkExMExhR2QyCkxDb3NDQTZYTU5HMmNQS1pmbEdiOXdrQ2dZQklDMFJ6T1F3Zk5NanRJMHYvNmx4UUZLY2lHeERSVEtSM2FQT1UKQ0V1cVZTT0o0bytHOWIydXAyc3R3RFBSSElqUEhJSS95S3FYeWtBeldJZE11MGROQU81K0tOWWRMWjhuSnBWVwpWelMyQWJFREhWRkRxOGd3M0xHVXMvNlN0NDE3cFNKb1Y4dkVXd0VldFpvb2pJZUpqbk1mSjhiZk12cHBRMDVHCnJGUFVkUUtCZ0NoTTdQQW1wclcrWVVOQlpMaTlYSngzeVh4MVhYWUs3NXU3cStjZHNudHhQNzBDb0FSd3hRK1AKOHlDUGFkNXNCbkI3RlNCUUZ1ajNyVmp1WTJxc0dkSjBoWkVwZmV0QXVsRm8wc01Xa2k2ZDI2ZHExaVZ0RlRobgovRzkydEI3OUYyYkczUk9uZEZ5QWV2ZEdicFBjbS9Nb0dkNUtCN3MzWTh4angway8ycThHCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==
+        client-certificate-data: {{client_cer}}
+        client-key-data: {{client_key}}
 kind: Secret
 metadata:
  name: kubeconfig
--- a/hack/deploy-karmada.sh
+++ b/hack/deploy-karmada.sh
@ -2,7 +2,18 @@

 set -o errexit
 set -o nounset
-set -o pipefail
+
+SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+CERT_DIR=${CERT_DIR:-"/var/run/karmada"}
+KARMADA_APISERVER_CONFIG="${CERT_DIR}/karmada-apiserver.config"
+KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
+KARMADA_KUBECONFIG="${KUBECONFIG_PATH}/karmada.config"
+etcd_replicas=1
+etcd_pod_label="etcd"
+apiserver_replicas=1
+apiserver_pod_label="karmada-apiserver"
+controller_replicas=1
+controller_pod_label="kube-controller-manager"

 function usage() {
  echo "This script will deploy karmada control plane to a cluster."
@ -10,7 +21,49 @@ function usage() {
  echo "Example: hack/deploy-karmada.sh"
 }

-SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+function waitPodReady() {
+    local pod_label=$1
+    local pod_namespaces=$2
+    local pod_replicas=$3
+
+    timeout=200
+    while [[ $timeout -gt 0 ]]; do
+        echo "Waiting for $pod_label pods to become Ready"
+        statuses=$(kubectl get pods -n $pod_namespaces -l app=$pod_label \
+         -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \
+          | grep "True" | wc -w)
+        if [[ $statuses -eq $pod_replicas ]]; then
+            break
+        else
+          sleep 1
+          (( timeout=timeout-1 ))
+        fi
+     done
+
+
+    if [[ $timeout -gt 0 ]]; then
+        echo "All $pod_label pods became Ready"
+    else
+        echo "ERROR: Not all $pod_label pods became Ready"
+        echo "kubectl get pods -l app=$pod_label"
+        kubectl get pods -l app=$pod_label
+        exit 1
+    fi
+}
+
+function installCRDs() {
+    if [ ! -f ${KARMADA_APISERVER_CONFIG} ]; then
+        echo "Please provide kubeconfig to connect karmada apiserver"
+        return 1
+    fi
+
+    # install APIs
+    kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/namespace.yaml"
+    kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/membercluster.karmada.io_memberclusters.yaml"
+    kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationpolicies.yaml"
+    kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationbindings.yaml"
+    kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationworks.yaml"
+}

 # create namespace for control plane components
 kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/namespace.yaml"
@ -20,14 +73,37 @@ kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/serviceaccount.yaml"
 kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/clusterrole.yaml"
 kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/clusterrolebinding.yaml"

-# install APIs
-kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/membercluster.karmada.io_memberclusters.yaml"
-kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationpolicies.yaml"
-kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationbindings.yaml"
-kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationworks.yaml"
+#generate cert
+"${SCRIPT_ROOT}"/hack/generate-cert.sh

-# create secret for controller-manager
-kubectl create secret generic kubeconfig --from-file=kubeconfig="${KUBECONFIG}" -n karmada-system
+# deploy karmada etcd
+kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/karmada-etcd.yaml"
+
+# Wait for karmada-etcd to come up before launching the rest of the components.
+waitPodReady $etcd_pod_label "karmada-system" $etcd_replicas
+
+# deploy karmada apiserver
+KARMADA_API_IP=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "karmada-control-plane")
+cp -rf ${SCRIPT_ROOT}/artifacts/deploy/karmada-apiserver.yaml ${SCRIPT_ROOT}/artifacts/deploy/karmada-apiserver-tmp.yaml
+sed -i "s/{{api_addr}}/${KARMADA_API_IP}/g" ${SCRIPT_ROOT}/artifacts/deploy/karmada-apiserver-tmp.yaml
+
+kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/karmada-apiserver-tmp.yaml"
+
+# Wait for karmada-apiserver to come up before launching the rest of the components.
+waitPodReady $apiserver_pod_label "karmada-system" $apiserver_replicas
+
+# deploy kube controller manager
+kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/kube-controller-manager.yaml"
+
+# Wait for karmada kube controller manager to come up before launching the rest of the components.
+waitPodReady $controller_pod_label "karmada-system" $controller_replicas
+
+export KUBECONFIG=${KARMADA_APISERVER_CONFIG}
+
+# install CRD APIs
+installCRDs
+
+export KUBECONFIG=${KARMADA_KUBECONFIG}

 # deploy controller-manager
 kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/controller-manager.yaml"
--- a/hack/generate-cert.sh
+++ b/hack/generate-cert.sh
@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+CERT_DIR=${CERT_DIR:-"/var/run/karmada"}
+mkdir -p "${CERT_DIR}" &>/dev/null || sudo mkdir -p "${CERT_DIR}"
+CFSSL_VERSION="v1.5.0"
+CONTROLPLANE_SUDO=$(test -w "${CERT_DIR}" || echo "sudo -E")
+ROOT_CA_FILE=${CERT_DIR}/server-ca.crt
+API_SECURE_PORT=${API_SECURE_PORT:-5443}
+
+# check whether openssl is installed.
+function ensure_openssl {
+    OPENSSL_BIN=$(command -v openssl)
+    if [[ ! -x ${OPENSSL_BIN} ]]; then
+      echo "Please install openssl and verify they are in \$PATH."
+      exit 1
+    fi
+}
+
+# downloads cfssl/cfssljson if they do not already exist in PATH
+function ensure-cfssl {
+    if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null; then
+        CFSSL_BIN=$(command -v cfssl)
+        CFSSLJSON_BIN=$(command -v cfssljson)
+        return 0
+    fi
+
+    # Install cfssl tools we need.
+    TEMP_PATH=$(mktemp -d)
+    pushd "${TEMP_PATH}" >/dev/null
+      GO111MODULE=on go get github.com/cloudflare/cfssl/cmd/...@"${CFSSL_VERSION}"
+    popd >/dev/null
+    rm -rf "${TEMP_PATH}"
+
+    GOPATH=$(go env | grep GOPATH | awk -F '=' '{print $2}'| sed 's/\"//g')
+    CFSSL_BIN="${GOPATH}/bin/cfssl"
+    CFSSLJSON_BIN="${GOPATH}/bin/cfssljson"
+    if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} ]]; then
+      echo "Failed to download 'cfssl'. Please install cfssl and cfssljson and verify they are in \$PATH."
+      echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..."
+      exit 1
+    fi
+}
+
+# creates a client CA, args are sudo, dest-dir, ca-id, purpose
+function create_signing_certkey {
+    local sudo=$1
+    local dest_dir=$2
+    local id=$3
+    local purpose=$4
+    # Create client ca
+    ${sudo} /usr/bin/env bash -e <<EOF
+    rm -f "${dest_dir}/${id}-ca.crt" "${dest_dir}/${id}-ca.key"
+    ${OPENSSL_BIN} req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "${dest_dir}/${id}-ca.key" -out "${dest_dir}/${id}-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
+    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment",${purpose}]}}}' > "${dest_dir}/${id}-ca-config.json"
+EOF
+}
+
+# signs a certificate: args are sudo, dest-dir, ca, filename (roughly), subject, hosts...
+function create_certkey {
+    local sudo=$1
+    local dest_dir=$2
+    local ca=$3
+    local id=$4
+    local cn=${5:-$4}
+    local hosts=""
+    local SEP=""
+    shift 5
+    while [ -n "${1:-}" ]; do
+        hosts+="${SEP}\"$1\""
+        SEP=","
+        shift 1
+    done
+    ${sudo} /usr/bin/env bash -e <<EOF
+    cd ${dest_dir}
+    echo '{"CN":"${cn}","hosts":[${hosts}],"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | ${CFSSL_BIN} gencert -ca=${ca}.crt -ca-key=${ca}.key -config=${ca}-config.json - | ${CFSSLJSON_BIN} -bare ${id}
+    mv "${id}-key.pem" "${id}.key"
+    mv "${id}.pem" "${id}.crt"
+    rm -f "${id}.csr"
+EOF
+}
+
+# create CA signers and certs
+function generate_certs {
+    # create CA signers
+    create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" server '"client auth","server auth"'
+
+    # signs a certificate
+    create_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "server-ca" karmada system:admin kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "localhost" "127.0.0.1"
+}
+
+# generate kubeconfig file for kubectl
+function generate_kubeconfig {
+    KARMADA_API_IP=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "karmada-control-plane")
+    write_client_kubeconfig "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "${ROOT_CA_FILE}" "${KARMADA_API_IP}" "${API_SECURE_PORT}" karmada-apiserver
+}
+
+# creates a self-contained kubeconfig: args are sudo, dest-dir, ca file, host, port, client id, token(optional)
+function write_client_kubeconfig {
+    local sudo=$1
+    local dest_dir=$2
+    local ca_file=$3
+    local api_host=$4
+    local api_port=$5
+    local client_id=$6
+    local token=${7:-}
+    cat <<EOF | ${sudo} tee "${dest_dir}"/"${client_id}".config > /dev/null
+apiVersion: v1
+kind: Config
+clusters:
+  - cluster:
+      "insecure-skip-tls-verify": true
+      server: https://${api_host}:${api_port}/
+    name: karmada-apiserver
+users:
+  - user:
+      token: ${token}
+      client-certificate: ${dest_dir}/karmada.crt
+      client-key: ${dest_dir}/karmada.key
+    name: karmada-apiserver
+contexts:
+  - context:
+      cluster: karmada-apiserver
+      user: karmada-apiserver
+    name: karmada-apiserver
+current-context: karmada-apiserver
+EOF
+}
+
+# generate a secret to store the certificates
+function generate_cert_secret {
+    local karmada_crt_file=${CERT_DIR}/karmada.crt
+    local karmada_key_file=${CERT_DIR}/karmada.key
+
+    sudo chmod 0644 ${karmada_crt_file}
+    sudo chmod 0644 ${karmada_key_file}
+
+    local karmada_ca=$(sudo cat ${ROOT_CA_FILE} | base64 | tr "\n" " "|sed s/[[:space:]]//g)
+    local karmada_crt=$(sudo cat ${karmada_crt_file} | base64 | tr "\n" " "|sed s/[[:space:]]//g)
+    local karmada_key=$(sudo cat ${karmada_key_file} | base64 | tr "\n" " "|sed s/[[:space:]]//g)
+
+    TEMP_PATH=$(mktemp -d)
+    cp -rf ${SCRIPT_ROOT}/artifacts/deploy/karmada-cert-secret.yaml ${TEMP_PATH}/karmada-cert-secret-tmp.yaml
+    cp -rf ${SCRIPT_ROOT}/artifacts/deploy/secret.yaml ${TEMP_PATH}/secret-tmp.yaml
+
+    sed -i "s/{{ca_crt}}/${karmada_ca}/g" ${TEMP_PATH}/karmada-cert-secret-tmp.yaml
+    sed -i "s/{{client_cer}}/${karmada_crt}/g" ${TEMP_PATH}/karmada-cert-secret-tmp.yaml
+    sed -i "s/{{client_key}}/${karmada_key}/g" ${TEMP_PATH}/karmada-cert-secret-tmp.yaml
+
+    sed -i "s/{{ca_crt}}/${karmada_ca}/g" ${TEMP_PATH}/secret-tmp.yaml
+    sed -i "s/{{client_cer}}/${karmada_crt}/g" ${TEMP_PATH}/secret-tmp.yaml
+    sed -i "s/{{client_key}}/${karmada_key}/g" ${TEMP_PATH}/secret-tmp.yaml
+
+    kubectl apply -f ${TEMP_PATH}/karmada-cert-secret-tmp.yaml
+    kubectl apply -f ${TEMP_PATH}/secret-tmp.yaml
+    rm -rf "${TEMP_PATH}"
+}
+
+ensure_openssl
+ensure-cfssl
+generate_certs
+generate_kubeconfig
+generate_cert_secret
--- a/hack/karmada-bootstrap.sh
+++ b/hack/karmada-bootstrap.sh
@ -10,12 +10,14 @@ set -o pipefail
 # 2. used by e2e testing to setup test environment automatically.

 KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
+KARMADA_APISERVER_KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG:-"/var/run/karmada"}

 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

 export KUBECONFIG_PATH="${KUBECONFIG_PATH}"
 "${REPO_ROOT}"/hack/local-up-karmada.sh
-export KUBECONFIG="${KUBECONFIG_PATH}/karmada.config"
+
+export KUBECONFIG="${KARMADA_APISERVER_KUBECONFIG}/karmada-apiserver.config"

 # Install karmadactl
 GO111MODULE=on go install "github.com/karmada-io/karmada/cmd/karmadactl"
--- a/hack/run-e2e.sh
+++ b/hack/run-e2e.sh
@ -10,15 +10,15 @@ set -o pipefail
 #
 # Usage: hack/run-e2e.sh
 # Example 1: hack/run-e2e.sh (run e2e with default config)
-# Example 2: export CONTROL_PLANE_KUBECONFIG=<KUBECONFIG PATH> hack/run-e2e.sh (run e2e with your KUBECONFIG)
+# Example 2: export KARMADA_APISERVER_KUBECONFIG=<KUBECONFIG PATH> hack/run-e2e.sh (run e2e with your KUBECONFIG)

-CONTROL_PLANE_KUBECONFIG=${CONTROL_PLANE_KUBECONFIG:-"${HOME}/.kube/karmada.config"}
+KARMADA_APISERVER_KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG:-"/var/run/karmada/karmada-apiserver.config"}

-export KUBECONFIG=${CONTROL_PLANE_KUBECONFIG}
+export KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG}

 # Install ginkgo
 GO111MODULE=on go install github.com/onsi/ginkgo/ginkgo

 # Run e2e
-export KUBECONFIG=${CONTROL_PLANE_KUBECONFIG}
+export KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG}
 ginkgo -v -race -failFast ./test/e2e/