Prepare etcd-server related tests for kube-apiserver

nodeup/pkg/model/kube_apiserver_test.go

@@ -158,6 +158,20 @@ func TestKubeAPIServerBuilder(t *testing.T) {
 	})
 }
 
+func TestDeddicatedAPIServerBuilder(t *testing.T) {
+	RunGoldenTest(t, "tests/golden/dedicated-apiserver", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error {
+		builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext}
+		return builder.Build(target)
+	})
+}
+
+func TestWithoutEtcdEventsAPIServerBuilder(t *testing.T) {
+	RunGoldenTest(t, "tests/golden/without-etcd-events", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error {
+		builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext}
+		return builder.Build(target)
+	})
+}
+
 func TestAwsIamAuthenticator(t *testing.T) {
 	RunGoldenTest(t, "tests/golden/awsiam", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error {
 		builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext}

nodeup/pkg/model/tests/golden/dedicated-apiserver/cluster.yaml

@@ -0,0 +1,67 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  name: minimal.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  etcdClusters:
+  - cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    memoryRequest: 100Mi
+    name: main
+    provider: Manager
+    backups:
+      backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main
+  - cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    memoryRequest: 100Mi
+    name: events
+    provider: Manager
+    backups:
+      backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
+  iam: {}
+  kubelet:
+    anonymousAuth: false
+  kubernetesVersion: v1.18.0
+  masterInternalName: api.internal.minimal.example.com
+  masterPublicName: api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+  - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: ami-1234
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a

nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml

@@ -0,0 +1,201 @@
+contents: |
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    annotations:
+      dns.alpha.kubernetes.io/external: api.minimal.example.com
+      dns.alpha.kubernetes.io/internal: api.internal.minimal.example.com
+      scheduler.alpha.kubernetes.io/critical-pod: ""
+    creationTimestamp: null
+    labels:
+      k8s-app: kube-apiserver
+    name: kube-apiserver
+    namespace: kube-system
+  spec:
+    containers:
+    - args:
+      - --allow-privileged=true
+      - --anonymous-auth=false
+      - --apiserver-count=1
+      - --authorization-mode=AlwaysAllow
+      - --bind-address=0.0.0.0
+      - --client-ca-file=/srv/kubernetes/ca.crt
+      - --cloud-config=/etc/kubernetes/cloud.config
+      - --cloud-provider=aws
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
+      - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt
+      - --etcd-certfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.crt
+      - --etcd-keyfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.key
+      - --etcd-servers-overrides=/events#https://127.0.0.1:4002
+      - --etcd-servers=https://127.0.0.1:4001
+      - --insecure-port=0
+      - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt
+      - --kubelet-client-key=/srv/kubernetes/kubelet-api.key
+      - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
+      - --proxy-client-cert-file=/srv/kubernetes/apiserver-aggregator.crt
+      - --proxy-client-key-file=/srv/kubernetes/apiserver-aggregator.key
+      - --requestheader-allowed-names=aggregator
+      - --requestheader-client-ca-file=/srv/kubernetes/apiserver-aggregator-ca.crt
+      - --requestheader-extra-headers-prefix=X-Remote-Extra-
+      - --requestheader-group-headers=X-Remote-Group
+      - --requestheader-username-headers=X-Remote-User
+      - --secure-port=443
+      - --service-account-key-file=/srv/kubernetes/service-account.key
+      - --service-cluster-ip-range=100.64.0.0/13
+      - --storage-backend=etcd3
+      - --tls-cert-file=/srv/kubernetes/server.crt
+      - --tls-private-key-file=/srv/kubernetes/server.key
+      - --v=2
+      - --logtostderr=false
+      - --alsologtostderr
+      - --log-file=/var/log/kube-apiserver.log
+      command:
+      - /usr/local/bin/kube-apiserver
+      image: k8s.gcr.io/kube-apiserver:v1.18.0
+      livenessProbe:
+        httpGet:
+          host: 127.0.0.1
+          path: /healthz
+          port: 443
+          scheme: HTTPS
+        initialDelaySeconds: 45
+        timeoutSeconds: 15
+      name: kube-apiserver
+      ports:
+      - containerPort: 443
+        hostPort: 443
+        name: https
+      resources:
+        requests:
+          cpu: 150m
+      volumeMounts:
+      - mountPath: /var/log/kube-apiserver.log
+        name: logfile
+      - mountPath: /etc/ssl
+        name: etcssl
+        readOnly: true
+      - mountPath: /etc/pki/tls
+        name: etcpkitls
+        readOnly: true
+      - mountPath: /etc/pki/ca-trust
+        name: etcpkica-trust
+        readOnly: true
+      - mountPath: /usr/share/ssl
+        name: usrsharessl
+        readOnly: true
+      - mountPath: /usr/ssl
+        name: usrssl
+        readOnly: true
+      - mountPath: /usr/lib/ssl
+        name: usrlibssl
+        readOnly: true
+      - mountPath: /usr/local/openssl
+        name: usrlocalopenssl
+        readOnly: true
+      - mountPath: /var/ssl
+        name: varssl
+        readOnly: true
+      - mountPath: /etc/openssl
+        name: etcopenssl
+        readOnly: true
+      - mountPath: /etc/kubernetes/pki/kube-apiserver
+        name: pki
+      - mountPath: /etc/kubernetes/cloud.config
+        name: cloudconfig
+        readOnly: true
+      - mountPath: /srv/kubernetes
+        name: srvkube
+        readOnly: true
+      - mountPath: /srv/sshproxy
+        name: srvsshproxy
+        readOnly: true
+    hostNetwork: true
+    priorityClassName: system-cluster-critical
+    tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    volumes:
+    - hostPath:
+        path: /var/log/kube-apiserver.log
+      name: logfile
+    - hostPath:
+        path: /etc/ssl
+      name: etcssl
+    - hostPath:
+        path: /etc/pki/tls
+      name: etcpkitls
+    - hostPath:
+        path: /etc/pki/ca-trust
+      name: etcpkica-trust
+    - hostPath:
+        path: /usr/share/ssl
+      name: usrsharessl
+    - hostPath:
+        path: /usr/ssl
+      name: usrssl
+    - hostPath:
+        path: /usr/lib/ssl
+      name: usrlibssl
+    - hostPath:
+        path: /usr/local/openssl
+      name: usrlocalopenssl
+    - hostPath:
+        path: /var/ssl
+      name: varssl
+    - hostPath:
+        path: /etc/openssl
+      name: etcopenssl
+    - hostPath:
+        path: /etc/kubernetes/pki/kube-apiserver
+        type: DirectoryOrCreate
+      name: pki
+    - hostPath:
+        path: /etc/kubernetes/cloud.config
+      name: cloudconfig
+    - hostPath:
+        path: /srv/kubernetes
+      name: srvkube
+    - hostPath:
+        path: /srv/sshproxy
+      name: srvsshproxy
+  status: {}
+path: /etc/kubernetes/manifests/kube-apiserver.manifest
+type: file
+---
+mode: "0755"
+path: /srv/kubernetes
+type: directory
+---
+contents:
+  task:
+    Name: kubelet-api
+    signer: ca
+    subject:
+      CommonName: kubelet-api
+    type: client
+mode: "0644"
+path: /srv/kubernetes/kubelet-api.crt
+type: file
+---
+contents:
+  task:
+    Name: kubelet-api
+    signer: ca
+    subject:
+      CommonName: kubelet-api
+    type: client
+mode: "0600"
+path: /srv/kubernetes/kubelet-api.key
+type: file
+---
+contents: ""
+ifNotExists: true
+mode: "0400"
+path: /var/log/kube-apiserver.log
+type: file
+---
+Name: kubelet-api
+signer: ca
+subject:
+  CommonName: kubelet-api
+type: client

nodeup/pkg/model/tests/golden/minimal/cluster.yaml

@@ -38,7 +38,7 @@ spec:
     kubenet: {}
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
-    - 0.0.0.0/0
+  - 0.0.0.0/0
   topology:
     masters: public
     nodes: public

nodeup/pkg/model/tests/golden/without-etcd-events/cluster.yaml

@@ -0,0 +1,67 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  name: minimal.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  etcdClusters:
+  - cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    memoryRequest: 100Mi
+    name: main
+    provider: Manager
+    backups:
+      backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main
+  - cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    memoryRequest: 100Mi
+    name: events
+    provider: Manager
+    backups:
+      backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
+  iam: {}
+  kubelet:
+    anonymousAuth: false
+  kubernetesVersion: v1.18.0
+  masterInternalName: api.internal.minimal.example.com
+  masterPublicName: api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+  - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: ami-1234
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a

nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml

@@ -0,0 +1,201 @@
+contents: |
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    annotations:
+      dns.alpha.kubernetes.io/external: api.minimal.example.com
+      dns.alpha.kubernetes.io/internal: api.internal.minimal.example.com
+      scheduler.alpha.kubernetes.io/critical-pod: ""
+    creationTimestamp: null
+    labels:
+      k8s-app: kube-apiserver
+    name: kube-apiserver
+    namespace: kube-system
+  spec:
+    containers:
+    - args:
+      - --allow-privileged=true
+      - --anonymous-auth=false
+      - --apiserver-count=1
+      - --authorization-mode=AlwaysAllow
+      - --bind-address=0.0.0.0
+      - --client-ca-file=/srv/kubernetes/ca.crt
+      - --cloud-config=/etc/kubernetes/cloud.config
+      - --cloud-provider=aws
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
+      - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt
+      - --etcd-certfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.crt
+      - --etcd-keyfile=/etc/kubernetes/pki/kube-apiserver/etcd-client.key
+      - --etcd-servers-overrides=/events#https://127.0.0.1:4002
+      - --etcd-servers=https://127.0.0.1:4001
+      - --insecure-port=0
+      - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt
+      - --kubelet-client-key=/srv/kubernetes/kubelet-api.key
+      - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
+      - --proxy-client-cert-file=/srv/kubernetes/apiserver-aggregator.crt
+      - --proxy-client-key-file=/srv/kubernetes/apiserver-aggregator.key
+      - --requestheader-allowed-names=aggregator
+      - --requestheader-client-ca-file=/srv/kubernetes/apiserver-aggregator-ca.crt
+      - --requestheader-extra-headers-prefix=X-Remote-Extra-
+      - --requestheader-group-headers=X-Remote-Group
+      - --requestheader-username-headers=X-Remote-User
+      - --secure-port=443
+      - --service-account-key-file=/srv/kubernetes/service-account.key
+      - --service-cluster-ip-range=100.64.0.0/13
+      - --storage-backend=etcd3
+      - --tls-cert-file=/srv/kubernetes/server.crt
+      - --tls-private-key-file=/srv/kubernetes/server.key
+      - --v=2
+      - --logtostderr=false
+      - --alsologtostderr
+      - --log-file=/var/log/kube-apiserver.log
+      command:
+      - /usr/local/bin/kube-apiserver
+      image: k8s.gcr.io/kube-apiserver:v1.18.0
+      livenessProbe:
+        httpGet:
+          host: 127.0.0.1
+          path: /healthz
+          port: 443
+          scheme: HTTPS
+        initialDelaySeconds: 45
+        timeoutSeconds: 15
+      name: kube-apiserver
+      ports:
+      - containerPort: 443
+        hostPort: 443
+        name: https
+      resources:
+        requests:
+          cpu: 150m
+      volumeMounts:
+      - mountPath: /var/log/kube-apiserver.log
+        name: logfile
+      - mountPath: /etc/ssl
+        name: etcssl
+        readOnly: true
+      - mountPath: /etc/pki/tls
+        name: etcpkitls
+        readOnly: true
+      - mountPath: /etc/pki/ca-trust
+        name: etcpkica-trust
+        readOnly: true
+      - mountPath: /usr/share/ssl
+        name: usrsharessl
+        readOnly: true
+      - mountPath: /usr/ssl
+        name: usrssl
+        readOnly: true
+      - mountPath: /usr/lib/ssl
+        name: usrlibssl
+        readOnly: true
+      - mountPath: /usr/local/openssl
+        name: usrlocalopenssl
+        readOnly: true
+      - mountPath: /var/ssl
+        name: varssl
+        readOnly: true
+      - mountPath: /etc/openssl
+        name: etcopenssl
+        readOnly: true
+      - mountPath: /etc/kubernetes/pki/kube-apiserver
+        name: pki
+      - mountPath: /etc/kubernetes/cloud.config
+        name: cloudconfig
+        readOnly: true
+      - mountPath: /srv/kubernetes
+        name: srvkube
+        readOnly: true
+      - mountPath: /srv/sshproxy
+        name: srvsshproxy
+        readOnly: true
+    hostNetwork: true
+    priorityClassName: system-cluster-critical
+    tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    volumes:
+    - hostPath:
+        path: /var/log/kube-apiserver.log
+      name: logfile
+    - hostPath:
+        path: /etc/ssl
+      name: etcssl
+    - hostPath:
+        path: /etc/pki/tls
+      name: etcpkitls
+    - hostPath:
+        path: /etc/pki/ca-trust
+      name: etcpkica-trust
+    - hostPath:
+        path: /usr/share/ssl
+      name: usrsharessl
+    - hostPath:
+        path: /usr/ssl
+      name: usrssl
+    - hostPath:
+        path: /usr/lib/ssl
+      name: usrlibssl
+    - hostPath:
+        path: /usr/local/openssl
+      name: usrlocalopenssl
+    - hostPath:
+        path: /var/ssl
+      name: varssl
+    - hostPath:
+        path: /etc/openssl
+      name: etcopenssl
+    - hostPath:
+        path: /etc/kubernetes/pki/kube-apiserver
+        type: DirectoryOrCreate
+      name: pki
+    - hostPath:
+        path: /etc/kubernetes/cloud.config
+      name: cloudconfig
+    - hostPath:
+        path: /srv/kubernetes
+      name: srvkube
+    - hostPath:
+        path: /srv/sshproxy
+      name: srvsshproxy
+  status: {}
+path: /etc/kubernetes/manifests/kube-apiserver.manifest
+type: file
+---
+mode: "0755"
+path: /srv/kubernetes
+type: directory
+---
+contents:
+  task:
+    Name: kubelet-api
+    signer: ca
+    subject:
+      CommonName: kubelet-api
+    type: client
+mode: "0644"
+path: /srv/kubernetes/kubelet-api.crt
+type: file
+---
+contents:
+  task:
+    Name: kubelet-api
+    signer: ca
+    subject:
+      CommonName: kubelet-api
+    type: client
+mode: "0600"
+path: /srv/kubernetes/kubelet-api.key
+type: file
+---
+contents: ""
+ifNotExists: true
+mode: "0400"
+path: /var/log/kube-apiserver.log
+type: file
+---
+Name: kubelet-api
+signer: ca
+subject:
+  CommonName: kubelet-api
+type: client