From 786538765a536eff36154782ecdd4270b183ac9a Mon Sep 17 00:00:00 2001 From: lihanbo Date: Tue, 15 Dec 2020 20:16:44 +0800 Subject: [PATCH] deploy karmada etcd and karmada apiserver --- README.md | 6 +- artifacts/deploy/karmada-apiserver.yaml | 128 ++++++++++++++ artifacts/deploy/karmada-cert-secret.yaml | 14 ++ artifacts/deploy/karmada-etcd.yaml | 127 +++++++++++++ artifacts/deploy/kube-controller-manager.yaml | 94 ++++++++++ artifacts/deploy/secret.yaml | 10 +- hack/deploy-karmada.sh | 94 +++++++++- hack/generate-cert.sh | 167 ++++++++++++++++++ hack/karmada-bootstrap.sh | 4 +- hack/run-e2e.sh | 8 +- 10 files changed, 630 insertions(+), 22 deletions(-) create mode 100644 artifacts/deploy/karmada-apiserver.yaml create mode 100644 artifacts/deploy/karmada-cert-secret.yaml create mode 100644 artifacts/deploy/karmada-etcd.yaml create mode 100644 artifacts/deploy/kube-controller-manager.yaml create mode 100755 hack/generate-cert.sh diff --git a/README.md b/README.md index 3b138eb81..50d805d8c 100644 --- a/README.md +++ b/README.md @@ -94,12 +94,14 @@ hack/create-cluster.sh member-cluster-1 /root/.kube/membercluster1.config ``` make karmadactl +export KUBECONFIG=/var/run/karmada/karmada-apiserver.config ./karmadactl join member-cluster-1 --member-cluster-kubeconfig=/root/.kube/membercluster1.config ``` 3. Verify member cluster is Joined to karmada successfully. ``` +export KUBECONFIG=/var/run/karmada/karmada-apiserver.config kubectl get membercluster -n karmada-cluster ``` @@ -108,14 +110,14 @@ kubectl get membercluster -n karmada-cluster 1. Create nginx deployment in karmada. ``` -export KUBECONFIG=/root/.kube/karmada.config +export KUBECONFIG=/var/run/karmada/karmada-apiserver.config kubectl create -f samples/nginx/deployment.yaml ``` 2. Create PropagationPolicy that will propagate nginx to member cluster. ``` -export KUBECONFIG=/root/.kube/karmada.config +export KUBECONFIG=/var/run/karmada/karmada-apiserver.config kubectl create -f samples/nginx/propagationpolicy.yaml ``` diff --git a/artifacts/deploy/karmada-apiserver.yaml b/artifacts/deploy/karmada-apiserver.yaml new file mode 100644 index 000000000..fd2972085 --- /dev/null +++ b/artifacts/deploy/karmada-apiserver.yaml @@ -0,0 +1,128 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karmada-apiserver + namespace: karmada-system + labels: + app: karmada-apiserver +spec: + replicas: 1 + selector: + matchLabels: + app: karmada-apiserver + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: karmada-apiserver + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - karmada-apiserver + topologyKey: kubernetes.io/hostname + containers: + - command: + - kube-apiserver + - --allow-privileged=true + - --authorization-mode=Node,RBAC + - --client-ca-file=/etc/kubernetes/pki/server-ca.crt + - --enable-admission-plugins=NodeRestriction + - --enable-bootstrap-token-auth=true + - --etcd-cafile=/etc/kubernetes/pki/server-ca.crt + - --etcd-certfile=/etc/kubernetes/pki/karmada.crt + - --etcd-keyfile=/etc/kubernetes/pki/karmada.key + - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379 + - --insecure-port=8080 + - --kubelet-client-certificate=/etc/kubernetes/pki/karmada.crt + - --kubelet-client-key=/etc/kubernetes/pki/karmada.key + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --runtime-config= + - --secure-port=5443 + - --service-cluster-ip-range=10.96.0.0/12 + - --proxy-client-cert-file=/etc/kubernetes/pki/karmada.crt + - --proxy-client-key-file=/etc/kubernetes/pki/karmada.key + - --requestheader-allowed-names=front-proxy-client + - --requestheader-client-ca-file=/etc/kubernetes/pki/server-ca.crt + - --requestheader-extra-headers-prefix=X-Remote-Extra- + - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + - --tls-cert-file=/etc/kubernetes/pki/karmada.crt + - --tls-private-key-file=/etc/kubernetes/pki/karmada.key + name: karmada-apiserver + image: k8s.gcr.io/kube-apiserver:v1.19.1 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 8 + httpGet: + host: {{api_addr}} + path: /livez + port: 5443 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + readinessProbe: + failureThreshold: 3 + httpGet: + host: {{api_addr}} + path: /readyz + port: 5443 + scheme: HTTPS + periodSeconds: 1 + successThreshold: 1 + timeoutSeconds: 15 + resources: + requests: + cpu: 250m + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/kubernetes/pki + name: k8s-certs + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + enableServiceLinks: true + hostNetwork: true + preemptionPolicy: PreemptLowerPriority + priority: 2000001000 + priorityClassName: system-node-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 + tolerations: + - effect: NoExecute + operator: Exists + volumes: + - name: k8s-certs + secret: + secretName: karmada-cert-secret + +--- + +apiVersion: v1 +kind: Service +metadata: + name: karmad-apiserver + namespace: karmada-system + labels: + app: karmada-apiserver +spec: + ports: + - name: karmad-apiserver-kubectl + port: 5443 + protocol: TCP + targetPort: 5443 + selector: + app: karmada-apiserver diff --git a/artifacts/deploy/karmada-cert-secret.yaml b/artifacts/deploy/karmada-cert-secret.yaml new file mode 100644 index 000000000..8d4e52c27 --- /dev/null +++ b/artifacts/deploy/karmada-cert-secret.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Secret +metadata: + name: karmada-cert-secret + namespace: karmada-system +type: Opaque +data: + server-ca.crt: | + {{ca_crt}} + karmada.crt: | + {{client_cer}} + karmada.key: | + {{client_key}} + diff --git a/artifacts/deploy/karmada-etcd.yaml b/artifacts/deploy/karmada-etcd.yaml new file mode 100644 index 000000000..d5b046914 --- /dev/null +++ b/artifacts/deploy/karmada-etcd.yaml @@ -0,0 +1,127 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: etcd + namespace: karmada-system + labels: + app: etcd +spec: + replicas: 1 + serviceName: etcd + selector: + matchLabels: + app: etcd + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: etcd + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - etcd + topologyKey: kubernetes.io/hostname + tolerations: + - operator: Exists + containers: + - name: etcd + image: k8s.gcr.io/etcd:3.4.13-0 + imagePullPolicy: Always + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - 'etcdctl get /registry --prefix --keys-only --endpoints https://127.0.0.1:2379 --cacert /etc/kubernetes/pki/etcd/server-ca.crt --cert /etc/kubernetes/pki/etcd/karmada.crt --key /etc/kubernetes/pki/etcd/karmada.key' + failureThreshold: 3 + initialDelaySeconds: 600 + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 10 + ports: + - containerPort: 2369 + name: client + protocol: TCP + - containerPort: 2370 + name: server + protocol: TCP + volumeMounts: + - mountPath: /var/lib/etcd + name: etcd-data + - mountPath: /etc/kubernetes/pki/etcd + name: etcd-certs + command: + - /usr/local/bin/etcd + - --name + - etcd0 + - --listen-peer-urls + - http://0.0.0.0:2380 + - --listen-client-urls + - https://0.0.0.0:2379 + - --advertise-client-urls + - https://etcd-client.karmada-system.svc.cluster.local:2379 + - --initial-cluster + - etcd0=http://etcd-0.etcd.karmada-system.svc.cluster.local:2380 + - --initial-cluster-state + - new + - --cert-file=/etc/kubernetes/pki/etcd/karmada.crt + - --client-cert-auth=true + - --key-file=/etc/kubernetes/pki/etcd/karmada.key + - --trusted-ca-file=/etc/kubernetes/pki/etcd/server-ca.crt + - --data-dir=/var/lib/etcd + volumes: + - hostPath: + path: /var/lib/karmada-etcd + type: DirectoryOrCreate + name: etcd-data + - name: etcd-certs + secret: + secretName: karmada-cert-secret +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + app: etcd + name: etcd-client + namespace: karmada-system +spec: + ports: + - name: etcd-client-port + port: 2379 + protocol: TCP + targetPort: 2379 + selector: + app: etcd + +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + app: etcd + name: etcd + namespace: karmada-system +spec: + ports: + - name: client + port: 2379 + protocol: TCP + targetPort: 2379 + - name: server + port: 2380 + protocol: TCP + targetPort: 2380 + clusterIP: None + selector: + app: etcd + diff --git a/artifacts/deploy/kube-controller-manager.yaml b/artifacts/deploy/kube-controller-manager.yaml new file mode 100644 index 000000000..633b82b1b --- /dev/null +++ b/artifacts/deploy/kube-controller-manager.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karmada-kube-controller-manager + namespace: karmada-system + labels: + app: kube-controller-manager +spec: + replicas: 1 + selector: + matchLabels: + app: kube-controller-manager + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: kube-controller-manager + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - kube-controller-manager + topologyKey: kubernetes.io/hostname + containers: + - command: + - kube-controller-manager + - --allocate-node-cidrs=true + - --authentication-kubeconfig=/etc/kubeconfig + - --authorization-kubeconfig=/etc/kubeconfig + - --bind-address=127.0.0.1 + - --client-ca-file=/etc/karmada/pki/server-ca.crt + - --cluster-cidr=10.244.0.0/16 + - --cluster-name=karmada + - --cluster-signing-cert-file=/etc/karmada/pki/server-ca.crt + - --cluster-signing-key-file=/etc/karmada/pki/server-ca.key + - --controllers=namespace,garbagecollector,serviceaccount-token + - --kubeconfig=/etc/kubeconfig + - --leader-elect=true + - --node-cidr-mask-size=24 + - --port=0 + - --root-ca-file=/etc/karmada/pki/server-ca.crt + - --service-account-private-key-file=/etc/karmada/pki/karmada.key + - --service-cluster-ip-range=10.96.0.0/12 + - --use-service-account-credentials=true + image: k8s.gcr.io/kube-controller-manager:v1.19.1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10257 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + name: kube-controller-manager + resources: + requests: + cpu: 200m + startupProbe: + failureThreshold: 24 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10257 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + volumeMounts: + - mountPath: /etc/karmada/pki + name: k8s-certs + readOnly: true + - mountPath: /etc/kubeconfig + subPath: kubeconfig + name: kubeconfig + priorityClassName: system-node-critical + volumes: + - name: k8s-certs + secret: + secretName: karmada-cert-secret + - name: kubeconfig + secret: + secretName: kubeconfig diff --git a/artifacts/deploy/secret.yaml b/artifacts/deploy/secret.yaml index c26f8d468..bc5024245 100644 --- a/artifacts/deploy/secret.yaml +++ b/artifacts/deploy/secret.yaml @@ -1,13 +1,11 @@ -# this is a example yaml. -# You need to replace value of kubeconfig with your own kubeconfig. apiVersion: v1 stringData: kubeconfig: |- apiVersion: v1 clusters: - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJd01USXhNVEEzTkRreU5Gb1hEVE13TVRJd09UQTNORGt5TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTGRZCnRSRGJkMEV1d3lZV3FmYy9jbDZLM0RqMi8vVmFnS1BlSUEwRWNaWENkemxybGFDWlNGNUFvVHE4TWpWbXlZRWYKc3VreG9TbTRJZEVMZDNYUzRwaHBIWGVRUllsMUxTbFJFRXNRcDBLbG1iOGhGa2JmVGppZjdwUUFyYTN2VlV1bAp1Uld3M3dJV0hROTA0WmxpZExoQ2hyYTZzb2pzM20xMXRDT2xGOEpIZTBFSjZkeWxqcjR5K1RORUVOQnJza0w2Cnd5UmlDTHJHbVo3Y2VxbjZiOWp1WFk0cjNiUGNvS3V1UWw4UHltTkQ4SmhBNEZYbldvblJ3ZERNRk9LTjlxQmoKL2dUejREVXMreFMyWEsxUlV1RVFCOGZ4blBtS29Ga0Q4QXBxTHFxcExWTUJjenVtMkdzSk9QYmlZT3FMTmd1awo4Ny9GUFRsT0RRSDhyVjdreWcwQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZIdjBReHZZaWc0ZWZoNk1kalJjZDlJQUJsemFNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDUHl3SmluMkJhc3hOdWp0WlZTaUd5WjIwSkVQVkI3MVc4cVYyS2ZoWHcvWnZpNGpDSApjWlBEYmxzdThqTFhraUgrY3ltWHpjc29CLzhkbWRUbWxuMmVhbjIzaHJhWUpVU1JqTDRNdlhGcTFNNmdxL1UwCkcvenhnaGw5WW5wYlVieGk0UFRuNVhaSFl5VDJ4cUEybW9JdkdmYytoV0NZMVpKMXRwaVRjZUFKY0d0QWxGdjkKdExEVHRiM0FhWW5WTVJkWFMxeDdRLzllV1VoeUFGV2JnQ0hrYjB0YXpQbTM0RGpuYXFSbU01dlV2VkkvTGJFYgpZN28vQ09reHJYekl3KzhGbTB4UXl0NUQ0Z3NrQThkNHVqQ1ZaVCtRT0ltdzRWeGtlQjdOZ3puN043MWNqQzZSCnQwZ2R5RUg1VU9STXBvT0d3VVVCTmpSdGd6cmUyWkYwUVlhOQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - server: https://172.17.0.2:6443 + certificate-authority-data: {{ca_crt}} + server: https://karmad-apiserver.karmada-system.svc.cluster.local:5443 name: kind-karmada contexts: - context: @@ -20,8 +18,8 @@ stringData: users: - name: kind-karmada user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lJQjRsVU9EaDVxa1V3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TURFeU1URXdOelE1TWpSYUZ3MHlNVEV5TVRFd056UTVNalphTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXFZd2o5N0QyZEp0UkhOM1MKUmdtODJyZ0hORE5IYkRHQndUV0Vxck9lQlFyS1JjUVBGWE5MbGJ1cXhvRlc0STdKNk8wd1RaRHBvNUxTNEZSOQptbmZLN2JXM040eno3aWpZRWNYeFFQWStWRFJ0TU9CWVJrdjhIenE3MjgvMXRMQTA5Z1RrK0Z0dFE5ckNjNEFTCnAxSjRNUEhEWHVXVC9TaDdTVkMwTHpPa0IrcWRoRVU5MzQrdkpmc1JCakhXd1IrNW1kaUxQWkFaNHRJZll0SHcKMmlNMkNmV09TeEVINTBackc3aW04aHB5czNNWWdwTE8yajc0K2FhZFZxZFI2QjlNWXJKMC8wbW4zTllLMU5rWgozbFVUdmRyd1NYUmV4YWZFUUVUeHZhekh4dE1aS2w4SlRrRHZqUjhSc1pvMUs5V0hvdkdsUEt1RnVqck41bHUvClFtVm83UUlEQVFBQm8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0h3WURWUjBqQkJnd0ZvQVVlL1JERzlpS0RoNStIb3gyTkZ4MzBnQUdYTm93RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFCN0VrdTM5TzBXN2RWRXdqc3dGYzdYNEtIWWZLQ3g4VCtCaGlHclVCcXhIcE40ckdCOFVoT0JLCnp3SWY1c2E2ZmhJUE82b2xXMlMwalFId1ZCZ1RpbTVhS25uaktObGIySHdLakNYUVBPZ0N2R0Z4c2M0WkxYVFEKVW5JQjcwVWZCUHJOakNFRFBsN3N1UXE0MWIycCtkVzRtV0dYQUNoYXdjWkJZbUtlZTVlNFBkTHdteW4zeWgxMgp6eEFpL1UxUm15c0FFd1ZqV25OMGhtQ3NTcWR0UTQ2eDNpT0gxR1MrbzB4MkUxM2RVMURZQk53d0pZTUh3WnVUCm5YSDRadXl1bjVESnRrNFlENDFFbm1NdFZ5OFhyTEVaR21LbDVjb1JMeWR6Sk4zSXRvY0Q2ZTdIYUJmcTZZRXUKZ0NiNG1RMWhhT2k3UVR0ZWpYTmFVekI5SER6aGdaTT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBcVl3ajk3RDJkSnRSSE4zU1JnbTgycmdITkROSGJER0J3VFdFcXJPZUJRcktSY1FQCkZYTkxsYnVxeG9GVzRJN0o2TzB3VFpEcG81TFM0RlI5bW5mSzdiVzNONHp6N2lqWUVjWHhRUFkrVkRSdE1PQlkKUmt2OEh6cTcyOC8xdExBMDlnVGsrRnR0UTlyQ2M0QVNwMUo0TVBIRFh1V1QvU2g3U1ZDMEx6T2tCK3FkaEVVOQozNCt2SmZzUkJqSFd3Uis1bWRpTFBaQVo0dElmWXRIdzJpTTJDZldPU3hFSDUwWnJHN2ltOGhweXMzTVlncExPCjJqNzQrYWFkVnFkUjZCOU1ZckowLzBtbjNOWUsxTmtaM2xVVHZkcndTWFJleGFmRVFFVHh2YXpIeHRNWktsOEoKVGtEdmpSOFJzWm8xSzlXSG92R2xQS3VGdWpyTjVsdS9RbVZvN1FJREFRQUJBb0lCQUYrMU56eUVEYXVuWFhOaApHR2ptNkUvRElIWHNHSDVQdUFKYjlxYnh1OXIyeTFYMHBkc2krV0R0emlvK0tzK2Fhd25za3A3R2xjejdmY1NhCmpVaWNKTlpwQktMOEUxcTJmYm9TdGV4STFNaHR5OUdTWXVKVDFGL0FwUGtoZkg3aUZSTm5rZVZNbnZKMTA5eWwKU0dKaG9HMW9uRE8rZnZxaVZtMzllNmxGaW12WXJwakQvNTFvWW1pd0JoRmpjYzcyMGlmdklmYURhNjBFd0wwcgpCaGViUG1mSFFZRzM3ZEZJeDN5dm4xWndXRVFma2hRcGYzeTlwNUNzOTJEYVBVVDhkR3BkZ3lNSDRTOWVMNlJrClBzbjdpcjNCNWtSZjJsSlVnMnV4eG5jbWVzQSsyZ0VYa3F2aldrc2d5Tk5MdGJ2S01La0dPTW5VU1h3MFBBc3kKa29wN0I3VUNnWUVBMGNMQ1BkVExVWjNBWFBrOWVsQkNUYnhnTndwcTlBSUJxQnVHQWhWSlpwaXpUR1NCeGIrcwo4bmtuZEhGd2d6cVcvdXp6RFZoaDl1MFpXQjF2eGdQdlJpN3B4Z3NIUG1qZjVPZko4KzdCQnhxQU1YdjZyTC9wCjNMYytpZVVYbTI1b1hjZjJtL3VOTitUVFBnOVVDWjlJOXhvUkxQTmQ4TlFabmxDUVd6aDlHaGNDZ1lFQXp1d08KeXR3MEQxdXBscml1Vm5XVWRTT3cwc1F3aEJiRnVsMzE5V2hibEFrS2pydGdiMXFMMXpnam1iZjlnR3RHK0FFWApvWVdsd2YxYUp5K0lKZWV0VndxUUZ1RXg4cmFuQ2xHWGNvSm15QmFFWlZBdFB6aUZWaHN0aXhHUHFLWUw3YUUrClBRTXFRTlZROUdLUExta093L1JpYnRWZUJ5ZjdISExoSzF1WmE1c0NnWUVBejFaL3diWnhNcitId1Y3c0VkYjcKY3ZOYzk0cm9wVURHZW5DYVNiell6UHpyWGZ1Syt2aTM3d1VxcHNMcXdBeE43TFl5bVdKZmswQlI2UE5QNUo5SApDRXllN3c4L25jVDBhc05pc0NlenpWMjRrVEZIV3pKbjY2K0Z0YkFwWVk0RXc0NUFpWFpnNUhyZkExMExhR2QyCkxDb3NDQTZYTU5HMmNQS1pmbEdiOXdrQ2dZQklDMFJ6T1F3Zk5NanRJMHYvNmx4UUZLY2lHeERSVEtSM2FQT1UKQ0V1cVZTT0o0bytHOWIydXAyc3R3RFBSSElqUEhJSS95S3FYeWtBeldJZE11MGROQU81K0tOWWRMWjhuSnBWVwpWelMyQWJFREhWRkRxOGd3M0xHVXMvNlN0NDE3cFNKb1Y4dkVXd0VldFpvb2pJZUpqbk1mSjhiZk12cHBRMDVHCnJGUFVkUUtCZ0NoTTdQQW1wclcrWVVOQlpMaTlYSngzeVh4MVhYWUs3NXU3cStjZHNudHhQNzBDb0FSd3hRK1AKOHlDUGFkNXNCbkI3RlNCUUZ1ajNyVmp1WTJxc0dkSjBoWkVwZmV0QXVsRm8wc01Xa2k2ZDI2ZHExaVZ0RlRobgovRzkydEI3OUYyYkczUk9uZEZ5QWV2ZEdicFBjbS9Nb0dkNUtCN3MzWTh4angway8ycThHCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + client-certificate-data: {{client_cer}} + client-key-data: {{client_key}} kind: Secret metadata: name: kubeconfig diff --git a/hack/deploy-karmada.sh b/hack/deploy-karmada.sh index 4780dccef..17f426be5 100755 --- a/hack/deploy-karmada.sh +++ b/hack/deploy-karmada.sh @@ -2,7 +2,18 @@ set -o errexit set -o nounset -set -o pipefail + +SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +CERT_DIR=${CERT_DIR:-"/var/run/karmada"} +KARMADA_APISERVER_CONFIG="${CERT_DIR}/karmada-apiserver.config" +KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} +KARMADA_KUBECONFIG="${KUBECONFIG_PATH}/karmada.config" +etcd_replicas=1 +etcd_pod_label="etcd" +apiserver_replicas=1 +apiserver_pod_label="karmada-apiserver" +controller_replicas=1 +controller_pod_label="kube-controller-manager" function usage() { echo "This script will deploy karmada control plane to a cluster." @@ -10,7 +21,49 @@ function usage() { echo "Example: hack/deploy-karmada.sh" } -SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +function waitPodReady() { + local pod_label=$1 + local pod_namespaces=$2 + local pod_replicas=$3 + + timeout=200 + while [[ $timeout -gt 0 ]]; do + echo "Waiting for $pod_label pods to become Ready" + statuses=$(kubectl get pods -n $pod_namespaces -l app=$pod_label \ + -o jsonpath='{.items[*].status.conditions[?(@.type=="Ready")].status}' \ + | grep "True" | wc -w) + if [[ $statuses -eq $pod_replicas ]]; then + break + else + sleep 1 + (( timeout=timeout-1 )) + fi + done + + + if [[ $timeout -gt 0 ]]; then + echo "All $pod_label pods became Ready" + else + echo "ERROR: Not all $pod_label pods became Ready" + echo "kubectl get pods -l app=$pod_label" + kubectl get pods -l app=$pod_label + exit 1 + fi +} + +function installCRDs() { + if [ ! -f ${KARMADA_APISERVER_CONFIG} ]; then + echo "Please provide kubeconfig to connect karmada apiserver" + return 1 + fi + + # install APIs + kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/namespace.yaml" + kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/membercluster.karmada.io_memberclusters.yaml" + kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationpolicies.yaml" + kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationbindings.yaml" + kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationworks.yaml" +} # create namespace for control plane components kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/namespace.yaml" @@ -20,14 +73,37 @@ kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/serviceaccount.yaml" kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/clusterrole.yaml" kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/clusterrolebinding.yaml" -# install APIs -kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/membercluster.karmada.io_memberclusters.yaml" -kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationpolicies.yaml" -kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationbindings.yaml" -kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/propagationstrategy.karmada.io_propagationworks.yaml" +#generate cert +"${SCRIPT_ROOT}"/hack/generate-cert.sh -# create secret for controller-manager -kubectl create secret generic kubeconfig --from-file=kubeconfig="${KUBECONFIG}" -n karmada-system +# deploy karmada etcd +kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/karmada-etcd.yaml" + +# Wait for karmada-etcd to come up before launching the rest of the components. +waitPodReady $etcd_pod_label "karmada-system" $etcd_replicas + +# deploy karmada apiserver +KARMADA_API_IP=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "karmada-control-plane") +cp -rf ${SCRIPT_ROOT}/artifacts/deploy/karmada-apiserver.yaml ${SCRIPT_ROOT}/artifacts/deploy/karmada-apiserver-tmp.yaml +sed -i "s/{{api_addr}}/${KARMADA_API_IP}/g" ${SCRIPT_ROOT}/artifacts/deploy/karmada-apiserver-tmp.yaml + +kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/karmada-apiserver-tmp.yaml" + +# Wait for karmada-apiserver to come up before launching the rest of the components. +waitPodReady $apiserver_pod_label "karmada-system" $apiserver_replicas + +# deploy kube controller manager +kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/kube-controller-manager.yaml" + +# Wait for karmada kube controller manager to come up before launching the rest of the components. +waitPodReady $controller_pod_label "karmada-system" $controller_replicas + +export KUBECONFIG=${KARMADA_APISERVER_CONFIG} + +# install CRD APIs +installCRDs + +export KUBECONFIG=${KARMADA_KUBECONFIG} # deploy controller-manager kubectl create -f "${SCRIPT_ROOT}/artifacts/deploy/controller-manager.yaml" diff --git a/hack/generate-cert.sh b/hack/generate-cert.sh new file mode 100755 index 000000000..d86a3f4d2 --- /dev/null +++ b/hack/generate-cert.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +CERT_DIR=${CERT_DIR:-"/var/run/karmada"} +mkdir -p "${CERT_DIR}" &>/dev/null || sudo mkdir -p "${CERT_DIR}" +CFSSL_VERSION="v1.5.0" +CONTROLPLANE_SUDO=$(test -w "${CERT_DIR}" || echo "sudo -E") +ROOT_CA_FILE=${CERT_DIR}/server-ca.crt +API_SECURE_PORT=${API_SECURE_PORT:-5443} + +# check whether openssl is installed. +function ensure_openssl { + OPENSSL_BIN=$(command -v openssl) + if [[ ! -x ${OPENSSL_BIN} ]]; then + echo "Please install openssl and verify they are in \$PATH." + exit 1 + fi +} + +# downloads cfssl/cfssljson if they do not already exist in PATH +function ensure-cfssl { + if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null; then + CFSSL_BIN=$(command -v cfssl) + CFSSLJSON_BIN=$(command -v cfssljson) + return 0 + fi + + # Install cfssl tools we need. + TEMP_PATH=$(mktemp -d) + pushd "${TEMP_PATH}" >/dev/null + GO111MODULE=on go get github.com/cloudflare/cfssl/cmd/...@"${CFSSL_VERSION}" + popd >/dev/null + rm -rf "${TEMP_PATH}" + + GOPATH=$(go env | grep GOPATH | awk -F '=' '{print $2}'| sed 's/\"//g') + CFSSL_BIN="${GOPATH}/bin/cfssl" + CFSSLJSON_BIN="${GOPATH}/bin/cfssljson" + if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} ]]; then + echo "Failed to download 'cfssl'. Please install cfssl and cfssljson and verify they are in \$PATH." + echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..." + exit 1 + fi +} + +# creates a client CA, args are sudo, dest-dir, ca-id, purpose +function create_signing_certkey { + local sudo=$1 + local dest_dir=$2 + local id=$3 + local purpose=$4 + # Create client ca + ${sudo} /usr/bin/env bash -e < "${dest_dir}/${id}-ca-config.json" +EOF +} + +# signs a certificate: args are sudo, dest-dir, ca, filename (roughly), subject, hosts... +function create_certkey { + local sudo=$1 + local dest_dir=$2 + local ca=$3 + local id=$4 + local cn=${5:-$4} + local hosts="" + local SEP="" + shift 5 + while [ -n "${1:-}" ]; do + hosts+="${SEP}\"$1\"" + SEP="," + shift 1 + done + ${sudo} /usr/bin/env bash -e < /dev/null +apiVersion: v1 +kind: Config +clusters: + - cluster: + "insecure-skip-tls-verify": true + server: https://${api_host}:${api_port}/ + name: karmada-apiserver +users: + - user: + token: ${token} + client-certificate: ${dest_dir}/karmada.crt + client-key: ${dest_dir}/karmada.key + name: karmada-apiserver +contexts: + - context: + cluster: karmada-apiserver + user: karmada-apiserver + name: karmada-apiserver +current-context: karmada-apiserver +EOF +} + +# generate a secret to store the certificates +function generate_cert_secret { + local karmada_crt_file=${CERT_DIR}/karmada.crt + local karmada_key_file=${CERT_DIR}/karmada.key + + sudo chmod 0644 ${karmada_crt_file} + sudo chmod 0644 ${karmada_key_file} + + local karmada_ca=$(sudo cat ${ROOT_CA_FILE} | base64 | tr "\n" " "|sed s/[[:space:]]//g) + local karmada_crt=$(sudo cat ${karmada_crt_file} | base64 | tr "\n" " "|sed s/[[:space:]]//g) + local karmada_key=$(sudo cat ${karmada_key_file} | base64 | tr "\n" " "|sed s/[[:space:]]//g) + + TEMP_PATH=$(mktemp -d) + cp -rf ${SCRIPT_ROOT}/artifacts/deploy/karmada-cert-secret.yaml ${TEMP_PATH}/karmada-cert-secret-tmp.yaml + cp -rf ${SCRIPT_ROOT}/artifacts/deploy/secret.yaml ${TEMP_PATH}/secret-tmp.yaml + + sed -i "s/{{ca_crt}}/${karmada_ca}/g" ${TEMP_PATH}/karmada-cert-secret-tmp.yaml + sed -i "s/{{client_cer}}/${karmada_crt}/g" ${TEMP_PATH}/karmada-cert-secret-tmp.yaml + sed -i "s/{{client_key}}/${karmada_key}/g" ${TEMP_PATH}/karmada-cert-secret-tmp.yaml + + sed -i "s/{{ca_crt}}/${karmada_ca}/g" ${TEMP_PATH}/secret-tmp.yaml + sed -i "s/{{client_cer}}/${karmada_crt}/g" ${TEMP_PATH}/secret-tmp.yaml + sed -i "s/{{client_key}}/${karmada_key}/g" ${TEMP_PATH}/secret-tmp.yaml + + kubectl apply -f ${TEMP_PATH}/karmada-cert-secret-tmp.yaml + kubectl apply -f ${TEMP_PATH}/secret-tmp.yaml + rm -rf "${TEMP_PATH}" +} + +ensure_openssl +ensure-cfssl +generate_certs +generate_kubeconfig +generate_cert_secret diff --git a/hack/karmada-bootstrap.sh b/hack/karmada-bootstrap.sh index f74e3a7ac..274f4a17f 100755 --- a/hack/karmada-bootstrap.sh +++ b/hack/karmada-bootstrap.sh @@ -10,12 +10,14 @@ set -o pipefail # 2. used by e2e testing to setup test environment automatically. KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} +KARMADA_APISERVER_KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG:-"/var/run/karmada"} REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. export KUBECONFIG_PATH="${KUBECONFIG_PATH}" "${REPO_ROOT}"/hack/local-up-karmada.sh -export KUBECONFIG="${KUBECONFIG_PATH}/karmada.config" + +export KUBECONFIG="${KARMADA_APISERVER_KUBECONFIG}/karmada-apiserver.config" # Install karmadactl GO111MODULE=on go install "github.com/karmada-io/karmada/cmd/karmadactl" diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index 6a2fb63a6..d87a6c5bb 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -10,15 +10,15 @@ set -o pipefail # # Usage: hack/run-e2e.sh # Example 1: hack/run-e2e.sh (run e2e with default config) -# Example 2: export CONTROL_PLANE_KUBECONFIG= hack/run-e2e.sh (run e2e with your KUBECONFIG) +# Example 2: export KARMADA_APISERVER_KUBECONFIG= hack/run-e2e.sh (run e2e with your KUBECONFIG) -CONTROL_PLANE_KUBECONFIG=${CONTROL_PLANE_KUBECONFIG:-"${HOME}/.kube/karmada.config"} +KARMADA_APISERVER_KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG:-"/var/run/karmada/karmada-apiserver.config"} -export KUBECONFIG=${CONTROL_PLANE_KUBECONFIG} +export KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG} # Install ginkgo GO111MODULE=on go install github.com/onsi/ginkgo/ginkgo # Run e2e -export KUBECONFIG=${CONTROL_PLANE_KUBECONFIG} +export KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG} ginkgo -v -race -failFast ./test/e2e/