Apply suggestions from code review

cmd/kops/create_cluster_integration_test.go

@@ -97,6 +97,11 @@ func TestCreateClusterComplex(t *testing.T) {
 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/complex", "v1alpha2")
 }
 
+// TestCreateClusterComplexPrivate runs kops create cluster, with a grab-bag of edge cases
+func TestCreateClusterComplexPrivate(t *testing.T) {
+	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/complex-private", "v1alpha2")
+}
+
 // TestCreateClusterHA runs kops create cluster ha.example.com --zones us-test-1a,us-test-1b,us-test-1c --master-zones us-test-1a,us-test-1b,us-test-1c
 func TestCreateClusterHA(t *testing.T) {
 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha", "v1alpha2")

tests/integration/create_cluster/complex-private/expected-v1alpha2.yaml

@@ -0,0 +1,287 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: complex.example.com
+spec:
+  additionalNetworkCIDRs:
+  - 10.1.0.0/16
+  - 10.2.0.0/16
+  - 10.3.0.0/16
+  - 10.4.0.0/16
+  api:
+    loadBalancer:
+      class: Network
+      type: Public
+  authorization:
+    rbac: {}
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://tests/complex.example.com
+  etcdClusters:
+  - cpuRequest: 200m
+    etcdMembers:
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1a
+      name: a
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1b
+      name: b
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1c
+      name: c
+    manager:
+      backupRetentionDays: 90
+    memoryRequest: 100Mi
+    name: main
+  - cpuRequest: 100m
+    etcdMembers:
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1a
+      name: a
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1b
+      name: b
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1c
+      name: c
+    manager:
+      backupRetentionDays: 90
+    memoryRequest: 100Mi
+    name: events
+  iam:
+    allowContainerRegistry: true
+    legacy: false
+  kubelet:
+    anonymousAuth: false
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  - ::/0
+  kubernetesVersion: v1.26.0
+  masterPublicName: api.complex.example.com
+  networkCIDR: 10.0.0.0/16
+  networking:
+    cni: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+  - 1.2.3.4/32
+  subnets:
+  - cidr: 10.0.64.0/18
+    name: us-test-1a
+    type: Private
+    zone: us-test-1a
+  - cidr: 10.0.128.0/18
+    name: us-test-1b
+    type: Private
+    zone: us-test-1b
+  - cidr: 10.0.192.0/18
+    name: us-test-1c
+    type: Private
+    zone: us-test-1c
+  - cidr: 10.1.0.0/16
+    name: us-test-1a-1
+    type: Private
+    zone: us-test-1a
+  - cidr: 10.2.0.0/16
+    name: us-test-1b-2
+    type: Private
+    zone: us-test-1b
+  - cidr: 10.3.0.0/16
+    name: us-test-1c-3
+    type: Private
+    zone: us-test-1c
+  - cidr: 10.4.0.0/16
+    name: us-test-1a-4
+    type: Private
+    zone: us-test-1a
+  - cidr: 10.0.0.0/21
+    name: utility-us-test-1a
+    type: Utility
+    zone: us-test-1a
+  - cidr: 10.0.24.0/21
+    name: utility-us-test-1b
+    type: Utility
+    zone: us-test-1b
+  - cidr: 10.0.40.0/21
+    name: utility-us-test-1c
+    type: Utility
+    zone: us-test-1c
+  - cidr: 10.0.8.0/21
+    name: utility-us-test-1a-1
+    type: Utility
+    zone: us-test-1a
+  - cidr: 10.0.32.0/21
+    name: utility-us-test-1b-2
+    type: Utility
+    zone: us-test-1b
+  - cidr: 10.0.48.0/21
+    name: utility-us-test-1c-3
+    type: Utility
+    zone: us-test-1c
+  - cidr: 10.0.16.0/21
+    name: utility-us-test-1a-4
+    type: Utility
+    zone: us-test-1a
+  topology:
+    bastion:
+      bastionPublicName: bastion.complex.example.com
+    dns:
+      type: Public
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: bastions
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpPutResponseHopLimit: 1
+    httpTokens: required
+  machineType: t2.micro
+  maxSize: 1
+  minSize: 1
+  role: Bastion
+  subnets:
+  - us-test-1a
+  - us-test-1b
+  - us-test-1c
+  - us-test-1a-1
+  - us-test-1b-2
+  - us-test-1c-3
+  - us-test-1a-4
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: control-plane-us-test-1a
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpTokens: required
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a
+  - us-test-1a-1
+  - us-test-1a-4
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: control-plane-us-test-1b
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpTokens: required
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1b
+  - us-test-1b-2
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: control-plane-us-test-1c
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpTokens: required
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1c
+  - us-test-1c-3
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: nodes-us-test-1a
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpPutResponseHopLimit: 1
+    httpTokens: required
+  machineType: t2.medium
+  maxSize: 4
+  minSize: 4
+  role: Node
+  subnets:
+  - us-test-1a
+  - us-test-1a-1
+  - us-test-1a-4
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: nodes-us-test-1b
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpPutResponseHopLimit: 1
+    httpTokens: required
+  machineType: t2.medium
+  maxSize: 3
+  minSize: 3
+  role: Node
+  subnets:
+  - us-test-1b
+  - us-test-1b-2
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: nodes-us-test-1c
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpPutResponseHopLimit: 1
+    httpTokens: required
+  machineType: t2.medium
+  maxSize: 3
+  minSize: 3
+  role: Node
+  subnets:
+  - us-test-1c
+  - us-test-1c-3

tests/integration/create_cluster/complex-private/options.yaml

@@ -0,0 +1,21 @@
+ClusterName: complex.example.com
+Zones:
+- us-test-1a
+- us-test-1b
+- us-test-1c
+CloudProvider: aws
+NetworkCIDRs:
+- 10.0.0.0/16
+- 10.1.0.0/16
+- 10.2.0.0/16
+- 10.3.0.0/16
+- 10.4.0.0/16
+Networking: cni
+Topology: private
+Bastion: true
+ControlPlaneCount: 3
+NodeCount: 10
+KubernetesVersion: v1.26.0
+# We specify SSHAccess but _not_ AdminAccess
+SSHAccess:
+- 1.2.3.4/32

tests/integration/create_cluster/complex/expected-v1alpha2.yaml

@@ -8,6 +8,7 @@ spec:
   - 10.1.0.0/16
   - 10.2.0.0/16
   - 10.3.0.0/16
+  - 10.4.0.0/16
   api:
     dns: {}
   authorization:
@@ -21,6 +22,12 @@ spec:
     - encryptedVolume: true
       instanceGroup: control-plane-us-test-1a
       name: a
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1b
+      name: b
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1c
+      name: c
     manager:
       backupRetentionDays: 90
     memoryRequest: 100Mi
@@ -30,6 +37,12 @@ spec:
     - encryptedVolume: true
       instanceGroup: control-plane-us-test-1a
       name: a
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1b
+      name: b
+    - encryptedVolume: true
+      instanceGroup: control-plane-us-test-1c
+      name: c
     manager:
       backupRetentionDays: 90
     memoryRequest: 100Mi
@@ -51,20 +64,32 @@ spec:
   sshAccess:
   - 1.2.3.4/32
   subnets:
-  - cidr: 10.0.0.0/16
+  - cidr: 10.0.0.0/18
     name: us-test-1a
     type: Public
     zone: us-test-1a
+  - cidr: 10.0.64.0/18
+    name: us-test-1b
+    type: Public
+    zone: us-test-1b
+  - cidr: 10.0.128.0/18
+    name: us-test-1c
+    type: Public
+    zone: us-test-1c
   - cidr: 10.1.0.0/16
     name: us-test-1a-1
     type: Public
     zone: us-test-1a
   - cidr: 10.2.0.0/16
-    name: us-test-1a-2
+    name: us-test-1b-2
     type: Public
-    zone: us-test-1a
+    zone: us-test-1b
   - cidr: 10.3.0.0/16
-    name: us-test-1a-3
+    name: us-test-1c-3
+    type: Public
+    zone: us-test-1c
+  - cidr: 10.4.0.0/16
+    name: us-test-1a-4
     type: Public
     zone: us-test-1a
   topology:
@@ -91,8 +116,49 @@ spec:
   subnets:
   - us-test-1a
   - us-test-1a-1
-  - us-test-1a-2
+  - us-test-1a-4
-  - us-test-1a-3
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: control-plane-us-test-1b
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpTokens: required
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1b
+  - us-test-1b-2
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: control-plane-us-test-1c
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpTokens: required
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1c
+  - us-test-1c-3
 
 ---
 
@@ -109,11 +175,54 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: t2.medium
-  maxSize: 1
+  maxSize: 4
-  minSize: 1
+  minSize: 4
   role: Node
   subnets:
   - us-test-1a
   - us-test-1a-1
-  - us-test-1a-2
+  - us-test-1a-4
-  - us-test-1a-3
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: nodes-us-test-1b
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpPutResponseHopLimit: 1
+    httpTokens: required
+  machineType: t2.medium
+  maxSize: 3
+  minSize: 3
+  role: Node
+  subnets:
+  - us-test-1b
+  - us-test-1b-2
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: complex.example.com
+  name: nodes-us-test-1c
+spec:
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230814
+  instanceMetadata:
+    httpPutResponseHopLimit: 1
+    httpTokens: required
+  machineType: t2.medium
+  maxSize: 3
+  minSize: 3
+  role: Node
+  subnets:
+  - us-test-1c
+  - us-test-1c-3

tests/integration/create_cluster/complex/options.yaml

@@ -1,13 +1,18 @@
 ClusterName: complex.example.com
 Zones:
 - us-test-1a
+- us-test-1b
+- us-test-1c
 CloudProvider: aws
 NetworkCIDRs:
 - 10.0.0.0/16
 - 10.1.0.0/16
 - 10.2.0.0/16
 - 10.3.0.0/16
+- 10.4.0.0/16
 Networking: cni
+ControlPlaneCount: 3
+NodeCount: 10
 KubernetesVersion: v1.26.0
 # We specify SSHAccess but _not_ AdminAccess
 SSHAccess: