From f4ec3df18783779f7c8650392e433a2514ab39ce Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Fri, 14 May 2021 15:40:56 +0300 Subject: [PATCH 1/2] Prepare etcd-server related tests for kube-apiserver --- nodeup/pkg/model/kube_apiserver_test.go | 14 ++ .../golden/dedicated-apiserver/cluster.yaml | 67 ++++++ .../tasks-kube-apiserver.yaml | 201 ++++++++++++++++++ .../model/tests/golden/minimal/cluster.yaml | 2 +- .../golden/without-etcd-events/cluster.yaml | 67 ++++++ .../tasks-kube-apiserver.yaml | 201 ++++++++++++++++++ 6 files changed, 551 insertions(+), 1 deletion(-) create mode 100644 nodeup/pkg/model/tests/golden/dedicated-apiserver/cluster.yaml create mode 100644 nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml create mode 100644 nodeup/pkg/model/tests/golden/without-etcd-events/cluster.yaml create mode 100644 nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml diff --git a/nodeup/pkg/model/kube_apiserver_test.go b/nodeup/pkg/model/kube_apiserver_test.go index b116818d8e..a6c0f653e6 100644 --- a/nodeup/pkg/model/kube_apiserver_test.go +++ b/nodeup/pkg/model/kube_apiserver_test.go @@ -158,6 +158,20 @@ func TestKubeAPIServerBuilder(t *testing.T) { }) } +func TestDeddicatedAPIServerBuilder(t *testing.T) { + RunGoldenTest(t, "tests/golden/dedicated-apiserver", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error { + builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext} + return builder.Build(target) + }) +} + +func TestWithoutEtcdEventsAPIServerBuilder(t *testing.T) { + RunGoldenTest(t, "tests/golden/without-etcd-events", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error { + builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext} + return builder.Build(target) + }) +} + func TestAwsIamAuthenticator(t *testing.T) { RunGoldenTest(t, "tests/golden/awsiam", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error { builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext} diff --git a/nodeup/pkg/model/tests/golden/dedicated-apiserver/cluster.yaml b/nodeup/pkg/model/tests/golden/dedicated-apiserver/cluster.yaml new file mode 100644 index 0000000000..23e74db736 --- /dev/null +++ b/nodeup/pkg/model/tests/golden/dedicated-apiserver/cluster.yaml @@ -0,0 +1,67 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + name: minimal.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + memoryRequest: 100Mi + name: main + provider: Manager + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + memoryRequest: 100Mi + name: events + provider: Manager + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events + iam: {} + kubelet: + anonymousAuth: false + kubernetesVersion: v1.18.0 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + name: master-us-test-1a + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: ami-1234 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a diff --git a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml new file mode 100644 index 0000000000..917e69ef52 --- /dev/null +++ b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml @@ -0,0 +1,201 @@ +contents: | + apiVersion: v1 + kind: Pod + metadata: + annotations: + dns.alpha.kubernetes.io/external: api.minimal.example.com + dns.alpha.kubernetes.io/internal: api.internal.minimal.example.com + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + k8s-app: kube-apiserver + name: kube-apiserver + namespace: kube-system + spec: + containers: + - args: + - --allow-privileged=true + - --anonymous-auth=false + - --apiserver-count=1 + - --authorization-mode=AlwaysAllow + - --bind-address=0.0.0.0 + - --client-ca-file=/srv/kubernetes/ca.crt + - --cloud-config=/etc/kubernetes/cloud.config + - --cloud-provider=aws + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota + - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt + - --etcd-certfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.crt + - --etcd-keyfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.key + - --etcd-servers-overrides=/events#https://127.0.0.1:4002 + - --etcd-servers=https://127.0.0.1:4001 + - --insecure-port=0 + - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt + - --kubelet-client-key=/srv/kubernetes/kubelet-api.key + - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP + - --proxy-client-cert-file=/srv/kubernetes/apiserver-aggregator.crt + - --proxy-client-key-file=/srv/kubernetes/apiserver-aggregator.key + - --requestheader-allowed-names=aggregator + - --requestheader-client-ca-file=/srv/kubernetes/apiserver-aggregator-ca.crt + - --requestheader-extra-headers-prefix=X-Remote-Extra- + - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + - --secure-port=443 + - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-cluster-ip-range=100.64.0.0/13 + - --storage-backend=etcd3 + - --tls-cert-file=/srv/kubernetes/server.crt + - --tls-private-key-file=/srv/kubernetes/server.key + - --v=2 + - --logtostderr=false + - --alsologtostderr + - --log-file=/var/log/kube-apiserver.log + command: + - /usr/local/bin/kube-apiserver + image: k8s.gcr.io/kube-apiserver:v1.18.0 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 443 + scheme: HTTPS + initialDelaySeconds: 45 + timeoutSeconds: 15 + name: kube-apiserver + ports: + - containerPort: 443 + hostPort: 443 + name: https + resources: + requests: + cpu: 150m + volumeMounts: + - mountPath: /var/log/kube-apiserver.log + name: logfile + - mountPath: /etc/ssl + name: etcssl + readOnly: true + - mountPath: /etc/pki/tls + name: etcpkitls + readOnly: true + - mountPath: /etc/pki/ca-trust + name: etcpkica-trust + readOnly: true + - mountPath: /usr/share/ssl + name: usrsharessl + readOnly: true + - mountPath: /usr/ssl + name: usrssl + readOnly: true + - mountPath: /usr/lib/ssl + name: usrlibssl + readOnly: true + - mountPath: /usr/local/openssl + name: usrlocalopenssl + readOnly: true + - mountPath: /var/ssl + name: varssl + readOnly: true + - mountPath: /etc/openssl + name: etcopenssl + readOnly: true + - mountPath: /etc/kubernetes/pki/kube-apiserver + name: pki + - mountPath: /etc/kubernetes/cloud.config + name: cloudconfig + readOnly: true + - mountPath: /srv/kubernetes + name: srvkube + readOnly: true + - mountPath: /srv/sshproxy + name: srvsshproxy + readOnly: true + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/log/kube-apiserver.log + name: logfile + - hostPath: + path: /etc/ssl + name: etcssl + - hostPath: + path: /etc/pki/tls + name: etcpkitls + - hostPath: + path: /etc/pki/ca-trust + name: etcpkica-trust + - hostPath: + path: /usr/share/ssl + name: usrsharessl + - hostPath: + path: /usr/ssl + name: usrssl + - hostPath: + path: /usr/lib/ssl + name: usrlibssl + - hostPath: + path: /usr/local/openssl + name: usrlocalopenssl + - hostPath: + path: /var/ssl + name: varssl + - hostPath: + path: /etc/openssl + name: etcopenssl + - hostPath: + path: /etc/kubernetes/pki/kube-apiserver + type: DirectoryOrCreate + name: pki + - hostPath: + path: /etc/kubernetes/cloud.config + name: cloudconfig + - hostPath: + path: /srv/kubernetes + name: srvkube + - hostPath: + path: /srv/sshproxy + name: srvsshproxy + status: {} +path: /etc/kubernetes/manifests/kube-apiserver.manifest +type: file +--- +mode: "0755" +path: /srv/kubernetes +type: directory +--- +contents: + task: + Name: kubelet-api + signer: ca + subject: + CommonName: kubelet-api + type: client +mode: "0644" +path: /srv/kubernetes/kubelet-api.crt +type: file +--- +contents: + task: + Name: kubelet-api + signer: ca + subject: + CommonName: kubelet-api + type: client +mode: "0600" +path: /srv/kubernetes/kubelet-api.key +type: file +--- +contents: "" +ifNotExists: true +mode: "0400" +path: /var/log/kube-apiserver.log +type: file +--- +Name: kubelet-api +signer: ca +subject: + CommonName: kubelet-api +type: client diff --git a/nodeup/pkg/model/tests/golden/minimal/cluster.yaml b/nodeup/pkg/model/tests/golden/minimal/cluster.yaml index e1b18d3dc1..23e74db736 100644 --- a/nodeup/pkg/model/tests/golden/minimal/cluster.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/cluster.yaml @@ -38,7 +38,7 @@ spec: kubenet: {} nonMasqueradeCIDR: 100.64.0.0/10 sshAccess: - - 0.0.0.0/0 + - 0.0.0.0/0 topology: masters: public nodes: public diff --git a/nodeup/pkg/model/tests/golden/without-etcd-events/cluster.yaml b/nodeup/pkg/model/tests/golden/without-etcd-events/cluster.yaml new file mode 100644 index 0000000000..23e74db736 --- /dev/null +++ b/nodeup/pkg/model/tests/golden/without-etcd-events/cluster.yaml @@ -0,0 +1,67 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + name: minimal.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + memoryRequest: 100Mi + name: main + provider: Manager + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + memoryRequest: 100Mi + name: events + provider: Manager + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events + iam: {} + kubelet: + anonymousAuth: false + kubernetesVersion: v1.18.0 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + name: master-us-test-1a + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: ami-1234 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a diff --git a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml new file mode 100644 index 0000000000..917e69ef52 --- /dev/null +++ b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml @@ -0,0 +1,201 @@ +contents: | + apiVersion: v1 + kind: Pod + metadata: + annotations: + dns.alpha.kubernetes.io/external: api.minimal.example.com + dns.alpha.kubernetes.io/internal: api.internal.minimal.example.com + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + k8s-app: kube-apiserver + name: kube-apiserver + namespace: kube-system + spec: + containers: + - args: + - --allow-privileged=true + - --anonymous-auth=false + - --apiserver-count=1 + - --authorization-mode=AlwaysAllow + - --bind-address=0.0.0.0 + - --client-ca-file=/srv/kubernetes/ca.crt + - --cloud-config=/etc/kubernetes/cloud.config + - --cloud-provider=aws + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota + - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt + - --etcd-certfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.crt + - --etcd-keyfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.key + - --etcd-servers-overrides=/events#https://127.0.0.1:4002 + - --etcd-servers=https://127.0.0.1:4001 + - --insecure-port=0 + - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt + - --kubelet-client-key=/srv/kubernetes/kubelet-api.key + - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP + - --proxy-client-cert-file=/srv/kubernetes/apiserver-aggregator.crt + - --proxy-client-key-file=/srv/kubernetes/apiserver-aggregator.key + - --requestheader-allowed-names=aggregator + - --requestheader-client-ca-file=/srv/kubernetes/apiserver-aggregator-ca.crt + - --requestheader-extra-headers-prefix=X-Remote-Extra- + - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + - --secure-port=443 + - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-cluster-ip-range=100.64.0.0/13 + - --storage-backend=etcd3 + - --tls-cert-file=/srv/kubernetes/server.crt + - --tls-private-key-file=/srv/kubernetes/server.key + - --v=2 + - --logtostderr=false + - --alsologtostderr + - --log-file=/var/log/kube-apiserver.log + command: + - /usr/local/bin/kube-apiserver + image: k8s.gcr.io/kube-apiserver:v1.18.0 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 443 + scheme: HTTPS + initialDelaySeconds: 45 + timeoutSeconds: 15 + name: kube-apiserver + ports: + - containerPort: 443 + hostPort: 443 + name: https + resources: + requests: + cpu: 150m + volumeMounts: + - mountPath: /var/log/kube-apiserver.log + name: logfile + - mountPath: /etc/ssl + name: etcssl + readOnly: true + - mountPath: /etc/pki/tls + name: etcpkitls + readOnly: true + - mountPath: /etc/pki/ca-trust + name: etcpkica-trust + readOnly: true + - mountPath: /usr/share/ssl + name: usrsharessl + readOnly: true + - mountPath: /usr/ssl + name: usrssl + readOnly: true + - mountPath: /usr/lib/ssl + name: usrlibssl + readOnly: true + - mountPath: /usr/local/openssl + name: usrlocalopenssl + readOnly: true + - mountPath: /var/ssl + name: varssl + readOnly: true + - mountPath: /etc/openssl + name: etcopenssl + readOnly: true + - mountPath: /etc/kubernetes/pki/kube-apiserver + name: pki + - mountPath: /etc/kubernetes/cloud.config + name: cloudconfig + readOnly: true + - mountPath: /srv/kubernetes + name: srvkube + readOnly: true + - mountPath: /srv/sshproxy + name: srvsshproxy + readOnly: true + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/log/kube-apiserver.log + name: logfile + - hostPath: + path: /etc/ssl + name: etcssl + - hostPath: + path: /etc/pki/tls + name: etcpkitls + - hostPath: + path: /etc/pki/ca-trust + name: etcpkica-trust + - hostPath: + path: /usr/share/ssl + name: usrsharessl + - hostPath: + path: /usr/ssl + name: usrssl + - hostPath: + path: /usr/lib/ssl + name: usrlibssl + - hostPath: + path: /usr/local/openssl + name: usrlocalopenssl + - hostPath: + path: /var/ssl + name: varssl + - hostPath: + path: /etc/openssl + name: etcopenssl + - hostPath: + path: /etc/kubernetes/pki/kube-apiserver + type: DirectoryOrCreate + name: pki + - hostPath: + path: /etc/kubernetes/cloud.config + name: cloudconfig + - hostPath: + path: /srv/kubernetes + name: srvkube + - hostPath: + path: /srv/sshproxy + name: srvsshproxy + status: {} +path: /etc/kubernetes/manifests/kube-apiserver.manifest +type: file +--- +mode: "0755" +path: /srv/kubernetes +type: directory +--- +contents: + task: + Name: kubelet-api + signer: ca + subject: + CommonName: kubelet-api + type: client +mode: "0644" +path: /srv/kubernetes/kubelet-api.crt +type: file +--- +contents: + task: + Name: kubelet-api + signer: ca + subject: + CommonName: kubelet-api + type: client +mode: "0600" +path: /srv/kubernetes/kubelet-api.key +type: file +--- +contents: "" +ifNotExists: true +mode: "0400" +path: /var/log/kube-apiserver.log +type: file +--- +Name: kubelet-api +signer: ca +subject: + CommonName: kubelet-api +type: client From 48ef1555bbaef906ce515570dbc30d53f944b04c Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Fri, 21 May 2021 18:53:34 +0300 Subject: [PATCH 2/2] Add etcd-server related tests for kube-apiserver --- .../model/tests/golden/dedicated-apiserver/cluster.yaml | 4 ++-- .../golden/dedicated-apiserver/tasks-kube-apiserver.yaml | 4 ++-- .../model/tests/golden/without-etcd-events/cluster.yaml | 9 --------- .../golden/without-etcd-events/tasks-kube-apiserver.yaml | 1 - 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/nodeup/pkg/model/tests/golden/dedicated-apiserver/cluster.yaml b/nodeup/pkg/model/tests/golden/dedicated-apiserver/cluster.yaml index 23e74db736..883c52a476 100644 --- a/nodeup/pkg/model/tests/golden/dedicated-apiserver/cluster.yaml +++ b/nodeup/pkg/model/tests/golden/dedicated-apiserver/cluster.yaml @@ -53,7 +53,7 @@ spec: apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: - name: master-us-test-1a + name: apiserver-us-test-1a labels: kops.k8s.io/cluster: minimal.example.com spec: @@ -62,6 +62,6 @@ spec: machineType: m3.medium maxSize: 1 minSize: 1 - role: Master + role: APIServer subnets: - us-test-1a diff --git a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml index 917e69ef52..d368426378 100644 --- a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml @@ -26,8 +26,8 @@ contents: | - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt - --etcd-certfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.crt - --etcd-keyfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.key - - --etcd-servers-overrides=/events#https://127.0.0.1:4002 - - --etcd-servers=https://127.0.0.1:4001 + - --etcd-servers-overrides=/events#https://events.etcd.minimal.example.com:4002 + - --etcd-servers=https://main.etcd.minimal.example.com:4001 - --insecure-port=0 - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt - --kubelet-client-key=/srv/kubernetes/kubelet-api.key diff --git a/nodeup/pkg/model/tests/golden/without-etcd-events/cluster.yaml b/nodeup/pkg/model/tests/golden/without-etcd-events/cluster.yaml index 23e74db736..f2f2d84e98 100644 --- a/nodeup/pkg/model/tests/golden/without-etcd-events/cluster.yaml +++ b/nodeup/pkg/model/tests/golden/without-etcd-events/cluster.yaml @@ -18,15 +18,6 @@ spec: provider: Manager backups: backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main - - cpuRequest: 100m - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - memoryRequest: 100Mi - name: events - provider: Manager - backups: - backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events iam: {} kubelet: anonymousAuth: false diff --git a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml index 917e69ef52..4c43f9a280 100644 --- a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml @@ -26,7 +26,6 @@ contents: | - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt - --etcd-certfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.crt - --etcd-keyfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.key - - --etcd-servers-overrides=/events#https://127.0.0.1:4002 - --etcd-servers=https://127.0.0.1:4001 - --insecure-port=0 - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt