Merge pull request #14127 from olemarkus/write-ig-input

Write the user provided IG spec to state store instead of the full spec

cmd/kops/create_cluster.go

@@ -40,6 +40,7 @@ import (
 
 	kopsbase "k8s.io/kops"
 	"k8s.io/kops/cmd/kops/util"
+	"k8s.io/kops/pkg/apis/kops"
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/registry"
 	kopsutil "k8s.io/kops/pkg/apis/kops/util"
@@ -68,9 +69,6 @@ type CreateClusterOptions struct {
 	NodeVolumeSize       int32
 	ContainerRuntime     string
 	OutDir               string
-	Image                string
-	NodeImage            string
-	MasterImage          string
 	DisableSubnetTags    bool
 	NetworkCIDR          string
 	DNSZone              string
@@ -549,22 +547,6 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr
 		}
 	}
 
-	if c.Image != "" {
-		for _, group := range instanceGroups {
-			group.Spec.Image = c.Image
-		}
-	}
-	if c.MasterImage != "" {
-		for _, group := range masters {
-			group.Spec.Image = c.MasterImage
-		}
-	}
-	if c.NodeImage != "" {
-		for _, group := range nodes {
-			group.Spec.Image = c.NodeImage
-		}
-	}
-
 	if c.AssociatePublicIP != nil {
 		for _, group := range instanceGroups {
 			group.Spec.AssociatePublicIP = c.AssociatePublicIP
@@ -662,19 +644,6 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr
 		return err
 	}
 
-	var fullInstanceGroups []*api.InstanceGroup
-	for _, group := range instanceGroups {
-		fullGroup, err := cloudup.PopulateInstanceGroupSpec(fullCluster, group, cloud, clusterResult.Channel)
-		if err != nil {
-			return err
-		}
-		fullGroup.AddInstanceGroupNodeLabel()
-		if cluster.Spec.GetCloudProvider() == api.CloudProviderGCE {
-			fullGroup.Spec.NodeLabels["cloud.google.com/metadata-proxy-ready"] = "true"
-		}
-		fullInstanceGroups = append(fullInstanceGroups, fullGroup)
-	}
-
 	kubernetesVersion, err := kopsutil.ParseKubernetesVersion(clusterResult.Cluster.Spec.KubernetesVersion)
 	if err != nil {
 		return fmt.Errorf("cannot parse KubernetesVersion %q in cluster: %w", clusterResult.Cluster.Spec.KubernetesVersion, err)
@@ -693,16 +662,28 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr
 		addons = append(addons, addon.Objects...)
 	}
 
-	err = validation.DeepValidate(fullCluster, fullInstanceGroups, true, nil)
+	{
-	if err != nil {
+		// Build full IG spec to ensure we end up with a valid IG
-		return err
+		fullInstanceGroups := []*kops.InstanceGroup{}
+		for _, group := range instanceGroups {
+			fullGroup, err := cloudup.PopulateInstanceGroupSpec(cluster, group, cloud, clusterResult.Channel)
+			if err != nil {
+				return err
+			}
+			fullInstanceGroups = append(fullInstanceGroups, fullGroup)
+		}
+
+		err = validation.DeepValidate(fullCluster, fullInstanceGroups, true, nil)
+		if err != nil {
+			return fmt.Errorf("validation of the full cluster and instance group specs failed: %w", err)
+		}
 	}
 
 	if c.DryRun {
 		var obj []runtime.Object
 		obj = append(obj, cluster)
 
-		for _, group := range fullInstanceGroups {
+		for _, group := range instanceGroups {
 			// Cluster name is not populated, and we need it
 			group.ObjectMeta.Labels = make(map[string]string)
 			group.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name
@@ -730,7 +711,7 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr
 	}
 
 	// Note we perform as much validation as we can, before writing a bad config
-	err = registry.CreateClusterConfig(ctx, clientset, cluster, fullInstanceGroups, addons)
+	err = registry.CreateClusterConfig(ctx, clientset, cluster, instanceGroups, addons)
 	if err != nil {
 		return fmt.Errorf("error writing updated configuration: %v", err)
 	}

pkg/apis/kops/validation/instancegroup_test.go

@@ -385,7 +385,7 @@ func TestIGUpdatePolicy(t *testing.T) {
 func TestValidInstanceGroup(t *testing.T) {
 	grid := []struct {
 		IG             *kops.InstanceGroup
-		ExpectedErrors int
+		ExpectedErrors []string
 		Description    string
 	}{
 		{
@@ -401,7 +401,7 @@ func TestValidInstanceGroup(t *testing.T) {
 					Image:   "my-image",
 				},
 			},
-			ExpectedErrors: 0,
+			ExpectedErrors: []string{},
 			Description:    "Valid master instance group failed to validate",
 		},
 		{
@@ -417,7 +417,7 @@ func TestValidInstanceGroup(t *testing.T) {
 					Image:   "my-image",
 				},
 			},
-			ExpectedErrors: 0,
+			ExpectedErrors: []string{},
 			Description:    "Valid API Server instance group failed to validate",
 		},
 		{
@@ -433,7 +433,7 @@ func TestValidInstanceGroup(t *testing.T) {
 					Image:   "my-image",
 				},
 			},
-			ExpectedErrors: 0,
+			ExpectedErrors: []string{},
 			Description:    "Valid node instance group failed to validate",
 		},
 		{
@@ -449,13 +449,28 @@ func TestValidInstanceGroup(t *testing.T) {
 					Image:   "my-image",
 				},
 			},
-			ExpectedErrors: 0,
+			ExpectedErrors: []string{},
 			Description:    "Valid bastion instance group failed to validate",
 		},
+		{
+			IG: &kops.InstanceGroup{
+				ObjectMeta: v1.ObjectMeta{
+					Name: "eu-central-1a",
+				},
+				Spec: kops.InstanceGroupSpec{
+					Role:    kops.InstanceGroupRoleBastion,
+					Subnets: []string{"eu-central-1a"},
+					MaxSize: fi.Int32(1),
+					MinSize: fi.Int32(1),
+				},
+			},
+			ExpectedErrors: []string{"Forbidden::spec.image"},
+			Description:    "Valid instance group must have image set",
+		},
 	}
 	for _, g := range grid {
 		errList := ValidateInstanceGroup(g.IG, nil, true)
-		testErrors(t, g.Description, errList, []string{})
+		testErrors(t, g.Description, errList, g.ExpectedErrors)
 	}
 }
 

tests/integration/create_cluster/complex/expected-v1alpha2.yaml

@@ -65,11 +65,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -86,11 +83,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/different-amis/expected-v1alpha2.yaml

@@ -77,11 +77,8 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: apiserver-us-test-1a
   role: APIServer
   subnets:
   - us-test-1a
@@ -101,11 +98,8 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: t2.micro
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: bastions
   role: Bastion
   subnets:
   - us-test-1a
@@ -125,11 +119,8 @@ spec:
     httpPutResponseHopLimit: 3
     httpTokens: required
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -149,11 +140,8 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml

@@ -66,12 +66,8 @@ metadata:
 spec:
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-1
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: master-us-test1-a
   role: Master
   subnets:
   - us-test1
@@ -90,12 +86,8 @@ metadata:
 spec:
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-2
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: nodes-us-test1-a
   role: Node
   subnets:
   - us-test1

tests/integration/create_cluster/ha/expected-v1alpha2.yaml

@@ -86,11 +86,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -107,11 +104,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1b
   role: Master
   subnets:
   - us-test-1b
@@ -128,11 +122,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1c
   role: Master
   subnets:
   - us-test-1c
@@ -149,11 +140,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a
@@ -170,11 +158,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1b
   role: Node
   subnets:
   - us-test-1b
@@ -191,11 +176,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1c
   role: Node
   subnets:
   - us-test-1c

tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml

@@ -86,11 +86,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -107,11 +104,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1b
   role: Master
   subnets:
   - us-test-1b
@@ -128,11 +122,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1c
   role: Master
   subnets:
   - us-test-1c
@@ -149,11 +140,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a
@@ -170,11 +158,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1b
   role: Node
   subnets:
   - us-test-1b
@@ -191,11 +176,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1c
   role: Node
   subnets:
   - us-test-1c

tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml

@@ -73,12 +73,8 @@ metadata:
 spec:
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-1
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: master-us-test1-a
   role: Master
   subnets:
   - us-test1
@@ -97,12 +93,8 @@ metadata:
 spec:
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-1
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: master-us-test1-b
   role: Master
   subnets:
   - us-test1
@@ -121,12 +113,8 @@ metadata:
 spec:
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-1
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: master-us-test1-c
   role: Master
   subnets:
   - us-test1
@@ -145,12 +133,8 @@ metadata:
 spec:
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-2
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: nodes-us-test1-a
   role: Node
   subnets:
   - us-test1
@@ -169,12 +153,8 @@ metadata:
 spec:
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-2
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: nodes-us-test1-b
   role: Node
   subnets:
   - us-test1
@@ -193,12 +173,8 @@ metadata:
 spec:
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-2
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: nodes-us-test1-c
   role: Node
   subnets:
   - us-test1

tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml

@@ -78,11 +78,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a-1
   role: Master
   subnets:
   - us-test-1a
@@ -99,11 +96,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a-2
   role: Master
   subnets:
   - us-test-1a
@@ -120,11 +114,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a-3
   role: Master
   subnets:
   - us-test-1a
@@ -141,11 +132,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml

@@ -94,11 +94,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a-1
   role: Master
   subnets:
   - us-test-1a
@@ -115,11 +112,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a-2
   role: Master
   subnets:
   - us-test-1a
@@ -136,11 +130,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a-3
   role: Master
   subnets:
   - us-test-1a
@@ -157,11 +148,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1b-1
   role: Master
   subnets:
   - us-test-1b
@@ -178,11 +166,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1b-2
   role: Master
   subnets:
   - us-test-1b
@@ -199,11 +184,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a
@@ -220,11 +202,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1b
   role: Node
   subnets:
   - us-test-1b

tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml

@@ -75,11 +75,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.micro
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: bastions
   role: Bastion
   subnets:
   - us-test-1a
@@ -96,11 +93,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -117,11 +111,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml

@@ -71,11 +71,8 @@ spec:
     httpPutResponseHopLimit: 3
     httpTokens: required
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -95,11 +92,8 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/karpenter/expected-v1alpha2.yaml

@@ -83,11 +83,8 @@ spec:
     httpPutResponseHopLimit: 3
     httpTokens: required
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -107,10 +104,6 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   manager: Karpenter
-  maxSize: 2
-  minSize: 2
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml

@@ -66,11 +66,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -87,11 +84,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml

@@ -66,11 +66,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -87,11 +84,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml

@@ -69,11 +69,8 @@ spec:
     httpPutResponseHopLimit: 3
     httpTokens: required
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -93,11 +90,8 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml

@@ -69,11 +69,8 @@ spec:
     httpPutResponseHopLimit: 3
     httpTokens: required
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -93,11 +90,8 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/minimal-1.24/expected-v1alpha2.yaml

@@ -69,11 +69,8 @@ spec:
     httpPutResponseHopLimit: 3
     httpTokens: required
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -93,11 +90,8 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/minimal-1.25/expected-v1alpha2.yaml

@@ -69,11 +69,8 @@ spec:
     httpPutResponseHopLimit: 3
     httpTokens: required
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -93,11 +90,8 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/minimal-1.26/expected-v1alpha2.yaml

@@ -69,11 +69,8 @@ spec:
     httpPutResponseHopLimit: 3
     httpTokens: required
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -93,11 +90,8 @@ spec:
     httpPutResponseHopLimit: 1
     httpTokens: required
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml

@@ -75,11 +75,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.micro
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: bastions
   role: Bastion
   subnets:
   - us-test-1a
@@ -96,11 +93,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -117,11 +111,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/overrides/expected-v1alpha2.yaml

@@ -69,11 +69,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -90,11 +87,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/private/expected-v1alpha2.yaml

@@ -78,11 +78,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.micro
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: bastions
   role: Bastion
   subnets:
   - us-test-1a
@@ -102,11 +99,8 @@ spec:
   - sg-exampleid4
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -126,11 +120,8 @@ spec:
   - sg-exampleid2
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml

@@ -73,12 +73,8 @@ metadata:
 spec:
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: f1-micro
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: bastions
   role: Bastion
   subnets:
   - us-test1
@@ -100,12 +96,8 @@ spec:
   - sg-exampleid4
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-1
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: master-us-test1-a
   role: Master
   subnets:
   - us-test1
@@ -127,12 +119,8 @@ spec:
   - sg-exampleid2
   image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118
   machineType: n1-standard-2
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    cloud.google.com/metadata-proxy-ready: "true"
-    kops.k8s.io/instancegroup: nodes-us-test1-a
   role: Node
   subnets:
   - us-test1

tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml

@@ -75,11 +75,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -96,11 +93,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml

@@ -68,11 +68,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -89,11 +86,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml

@@ -68,11 +68,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -89,11 +86,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml

@@ -67,11 +67,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: m3.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: master-us-test-1a
   role: Master
   subnets:
   - us-test-1a
@@ -88,11 +85,8 @@ metadata:
 spec:
   image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615
   machineType: t2.medium
-  manager: CloudGroup
   maxSize: 1
   minSize: 1
-  nodeLabels:
-    kops.k8s.io/instancegroup: nodes-us-test-1a
   role: Node
   subnets:
   - us-test-1a

upup/pkg/fi/cloudup/apply_cluster.go

@@ -819,6 +819,7 @@ func (c *ApplyClusterCmd) upgradeSpecs(assetBuilder *assets.AssetBuilder) error
 	}
 	c.Cluster = fullCluster
 
+	klog.Infof("Populating instance group from upgradeSpecs")
 	for i, g := range c.InstanceGroups {
 		fullGroup, err := PopulateInstanceGroupSpec(fullCluster, g, c.Cloud, c.channel)
 		if err != nil {

upup/pkg/fi/cloudup/new_cluster.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/blang/semver/v4"
 	"k8s.io/apimachinery/pkg/api/resource"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -30,7 +31,9 @@ import (
 
 	"k8s.io/kops"
 	api "k8s.io/kops/pkg/apis/kops"
+	kopsapi "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/model"
+	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/client/simple"
 	"k8s.io/kops/pkg/dns"
 	"k8s.io/kops/pkg/featureflag"
@@ -40,6 +43,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/azure"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
+	"k8s.io/kops/util/pkg/architectures"
 	"k8s.io/kops/util/pkg/vfs"
 )
 
@@ -150,6 +154,10 @@ type NewClusterOptions struct {
 
 	// InstanceManager specifies which manager to use for managing instances.
 	InstanceManager string
+
+	Image       string
+	NodeImage   string
+	MasterImage string
 }
 
 func (o *NewClusterOptions) InitDefaults() {
@@ -260,9 +268,18 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
 			return nil, fmt.Errorf("unable to infer cloud provider from zones. pass in the cloud provider explicitly using --cloud")
 		}
 	}
+
+	var cloud fi.Cloud
+
 	switch api.CloudProviderID(opt.CloudProvider) {
 	case api.CloudProviderAWS:
 		cluster.Spec.CloudProvider.AWS = &api.AWSSpec{}
+		cloudTags := map[string]string{}
+		awsCloud, err := awsup.NewAWSCloud(opt.Zones[0][:len(opt.Zones[0])-1], cloudTags)
+		if err != nil {
+			return nil, err
+		}
+		cloud = awsCloud
 	case api.CloudProviderAzure:
 		cluster.Spec.CloudProvider.Azure = &api.AzureSpec{
 			SubscriptionID:    opt.AzureSubscriptionID,
@@ -310,7 +327,7 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
 		}
 	}
 
-	err = setupVPC(opt, &cluster)
+	err = setupVPC(opt, &cluster, cloud)
 	if err != nil {
 		return nil, err
 	}
@@ -373,6 +390,96 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
 	instanceGroups = append(instanceGroups, nodes...)
 	instanceGroups = append(instanceGroups, bastions...)
 
+	for _, instanceGroup := range instanceGroups {
+		g := instanceGroup
+		ig := g
+		if instanceGroup.Spec.Image == "" {
+			if opt.Image != "" {
+				instanceGroup.Spec.Image = opt.Image
+			} else {
+				architecture, err := MachineArchitecture(cloud, instanceGroup.Spec.MachineType)
+				if err != nil {
+					return nil, err
+				}
+				instanceGroup.Spec.Image = defaultImage(&cluster, channel, architecture)
+			}
+		}
+
+		// TODO: Clean up
+		if g.IsMaster() {
+			if g.Spec.MachineType == "" {
+				g.Spec.MachineType, err = defaultMachineType(cloud, &cluster, ig)
+				if err != nil {
+					return nil, fmt.Errorf("error assigning default machine type for masters: %v", err)
+				}
+
+			}
+		} else if g.Spec.Role == kopsapi.InstanceGroupRoleBastion {
+			if g.Spec.MachineType == "" {
+				g.Spec.MachineType, err = defaultMachineType(cloud, &cluster, g)
+				if err != nil {
+					return nil, fmt.Errorf("error assigning default machine type for bastions: %v", err)
+				}
+			}
+		} else {
+			if g.IsAPIServerOnly() && !featureflag.APIServerNodes.Enabled() {
+				return nil, fmt.Errorf("apiserver nodes requires the APIServerNodes feature flag to be enabled")
+			}
+			if g.Spec.MachineType == "" {
+				g.Spec.MachineType, err = defaultMachineType(cloud, &cluster, g)
+				if err != nil {
+					return nil, fmt.Errorf("error assigning default machine type for nodes: %v", err)
+				}
+			}
+
+		}
+
+		if ig.Spec.Tenancy != "" && ig.Spec.Tenancy != "default" {
+			switch cluster.Spec.GetCloudProvider() {
+			case kopsapi.CloudProviderAWS:
+				if _, ok := awsDedicatedInstanceExceptions[g.Spec.MachineType]; ok {
+					return nil, fmt.Errorf("invalid dedicated instance type: %s", g.Spec.MachineType)
+				}
+			default:
+				klog.Warning("Trying to set tenancy on non-AWS environment")
+			}
+		}
+
+		if ig.IsMaster() {
+			if len(ig.Spec.Subnets) == 0 {
+				return nil, fmt.Errorf("master InstanceGroup %s did not specify any Subnets", g.ObjectMeta.Name)
+			}
+		} else if ig.IsAPIServerOnly() && cluster.Spec.IsIPv6Only() {
+			if len(ig.Spec.Subnets) == 0 {
+				for _, subnet := range cluster.Spec.Subnets {
+					if subnet.Type != kopsapi.SubnetTypePrivate && subnet.Type != kopsapi.SubnetTypeUtility {
+						ig.Spec.Subnets = append(g.Spec.Subnets, subnet.Name)
+					}
+				}
+			}
+		} else {
+			if len(ig.Spec.Subnets) == 0 {
+				for _, subnet := range cluster.Spec.Subnets {
+					if subnet.Type != kopsapi.SubnetTypeDualStack && subnet.Type != kopsapi.SubnetTypeUtility {
+						g.Spec.Subnets = append(g.Spec.Subnets, subnet.Name)
+					}
+				}
+			}
+
+			if len(g.Spec.Subnets) == 0 {
+				for _, subnet := range cluster.Spec.Subnets {
+					if subnet.Type != kopsapi.SubnetTypeUtility {
+						g.Spec.Subnets = append(g.Spec.Subnets, subnet.Name)
+					}
+				}
+			}
+		}
+
+		if len(g.Spec.Subnets) == 0 {
+			return nil, fmt.Errorf("unable to infer any Subnets for InstanceGroup %s ", g.ObjectMeta.Name)
+		}
+	}
+
 	result := NewClusterResult{
 		Cluster:        &cluster,
 		InstanceGroups: instanceGroups,
@@ -381,17 +488,13 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
 	return &result, nil
 }
 
-func setupVPC(opt *NewClusterOptions, cluster *api.Cluster) error {
+func setupVPC(opt *NewClusterOptions, cluster *api.Cluster, cloud fi.Cloud) error {
 	cluster.Spec.NetworkID = opt.NetworkID
 
 	switch cluster.Spec.GetCloudProvider() {
 	case api.CloudProviderAWS:
 		if cluster.Spec.NetworkID == "" && len(opt.SubnetIDs) > 0 {
-			cloudTags := map[string]string{}
+			awsCloud := cloud.(awsup.AWSCloud)
-			awsCloud, err := awsup.NewAWSCloud(opt.Zones[0][:len(opt.Zones[0])-1], cloudTags)
-			if err != nil {
-				return fmt.Errorf("error loading cloud: %v", err)
-			}
 			res, err := awsCloud.EC2().DescribeSubnets(&ec2.DescribeSubnetsInput{
 				SubnetIds: []*string{aws.String(opt.SubnetIDs[0])},
 			})
@@ -755,6 +858,8 @@ func setupMasters(opt *NewClusterOptions, cluster *api.Cluster, zoneToSubnetMap
 				}
 			}
 
+			g.Spec.Image = opt.MasterImage
+
 			masters = append(masters, g)
 		}
 	}
@@ -880,6 +985,8 @@ func setupNodes(opt *NewClusterOptions, cluster *api.Cluster, zoneToSubnetMap ma
 			}
 		}
 
+		g.Spec.Image = opt.NodeImage
+
 		nodes = append(nodes, g)
 	}
 
@@ -1114,6 +1221,8 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S
 			bastionGroup := &api.InstanceGroup{}
 			bastionGroup.Spec.Role = api.InstanceGroupRoleBastion
 			bastionGroup.ObjectMeta.Name = "bastions"
+			bastionGroup.Spec.MaxSize = fi.Int32(1)
+			bastionGroup.Spec.MinSize = fi.Int32(1)
 			bastions = append(bastions, bastionGroup)
 
 			if !dns.IsGossipHostname(cluster.Name) {
@@ -1321,3 +1430,63 @@ func addCiliumNetwork(cluster *api.Cluster) {
 	enabled := false
 	cluster.Spec.KubeProxy.Enabled = &enabled
 }
+
+// defaultImage returns the default Image, based on the cloudprovider
+func defaultImage(cluster *kopsapi.Cluster, channel *kopsapi.Channel, architecture architectures.Architecture) string {
+	if channel != nil {
+		var kubernetesVersion *semver.Version
+		if cluster.Spec.KubernetesVersion != "" {
+			var err error
+			kubernetesVersion, err = util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
+			if err != nil {
+				klog.Warningf("cannot parse KubernetesVersion %q in cluster", cluster.Spec.KubernetesVersion)
+			}
+		}
+		if kubernetesVersion != nil {
+			image := channel.FindImage(cluster.Spec.GetCloudProvider(), *kubernetesVersion, architecture)
+			if image != nil {
+				return image.Name
+			}
+		}
+	}
+
+	switch cluster.Spec.GetCloudProvider() {
+	case kopsapi.CloudProviderDO:
+		return defaultDONodeImage
+	}
+	klog.Infof("Cannot set default Image for CloudProvider=%q", cluster.Spec.GetCloudProvider())
+	return ""
+}
+
+func MachineArchitecture(cloud fi.Cloud, machineType string) (architectures.Architecture, error) {
+	if machineType == "" {
+		return architectures.ArchitectureAmd64, nil
+	}
+
+	switch cloud.ProviderID() {
+	case kopsapi.CloudProviderAWS:
+		info, err := cloud.(awsup.AWSCloud).DescribeInstanceType(machineType)
+		if err != nil {
+			return "", fmt.Errorf("error finding instance info for instance type %q: %v", machineType, err)
+		}
+		if info.ProcessorInfo == nil || len(info.ProcessorInfo.SupportedArchitectures) == 0 {
+			return "", fmt.Errorf("error finding architecture info for instance type %q", machineType)
+		}
+		var unsupported []string
+		for _, arch := range info.ProcessorInfo.SupportedArchitectures {
+			// Return the first found supported architecture, in order of popularity
+			switch fi.StringValue(arch) {
+			case ec2.ArchitectureTypeX8664:
+				return architectures.ArchitectureAmd64, nil
+			case ec2.ArchitectureTypeArm64:
+				return architectures.ArchitectureArm64, nil
+			default:
+				unsupported = append(unsupported, fi.StringValue(arch))
+			}
+		}
+		return "", fmt.Errorf("unsupported architecture for instance type %q: %v", machineType, unsupported)
+	default:
+		// No other clouds are known to support any other architectures at this time
+		return architectures.ArchitectureAmd64, nil
+	}
+}

upup/pkg/fi/cloudup/populate_instancegroup_spec.go

@@ -20,18 +20,13 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/blang/semver/v4"
 	"k8s.io/klog/v2"
 
 	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/apis/kops/validation"
-	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
-	"k8s.io/kops/util/pkg/architectures"
 	"k8s.io/kops/util/pkg/reflectutils"
 )
 
@@ -67,117 +62,17 @@ var awsDedicatedInstanceExceptions = map[string]bool{
 // PopulateInstanceGroupSpec sets default values in the InstanceGroup
 // The InstanceGroup is simpler than the cluster spec, so we just populate in place (like the rest of k8s)
 func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup, cloud fi.Cloud, channel *kops.Channel) (*kops.InstanceGroup, error) {
+	klog.Infof("Populating instance group spec for %q", input.GetName())
+
 	var err error
 	err = validation.ValidateInstanceGroup(input, nil, false).ToAggregate()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed validating input specs: %w", err)
 	}
 
 	ig := &kops.InstanceGroup{}
 	reflectutils.JSONMergeStruct(ig, input)
 
-	// TODO: Clean up
-	if ig.IsMaster() {
-		if ig.Spec.MachineType == "" {
-			ig.Spec.MachineType, err = defaultMachineType(cloud, cluster, ig)
-			if err != nil {
-				return nil, fmt.Errorf("error assigning default machine type for masters: %v", err)
-			}
-
-		}
-		if ig.Spec.MinSize == nil {
-			ig.Spec.MinSize = fi.Int32(1)
-		}
-		if ig.Spec.MaxSize == nil {
-			ig.Spec.MaxSize = fi.Int32(1)
-		}
-	} else if ig.Spec.Role == kops.InstanceGroupRoleBastion {
-		if ig.Spec.MachineType == "" {
-			ig.Spec.MachineType, err = defaultMachineType(cloud, cluster, ig)
-			if err != nil {
-				return nil, fmt.Errorf("error assigning default machine type for bastions: %v", err)
-			}
-		}
-		if ig.Spec.MinSize == nil {
-			ig.Spec.MinSize = fi.Int32(1)
-		}
-		if ig.Spec.MaxSize == nil {
-			ig.Spec.MaxSize = fi.Int32(1)
-		}
-	} else {
-		if ig.IsAPIServerOnly() && !featureflag.APIServerNodes.Enabled() {
-			return nil, fmt.Errorf("apiserver nodes requires the APIServerNodes feature flag to be enabled")
-		}
-		if ig.Spec.MachineType == "" {
-			ig.Spec.MachineType, err = defaultMachineType(cloud, cluster, ig)
-			if err != nil {
-				return nil, fmt.Errorf("error assigning default machine type for nodes: %v", err)
-			}
-		}
-		if ig.Spec.MinSize == nil {
-			ig.Spec.MinSize = fi.Int32(2)
-		}
-		if ig.Spec.MaxSize == nil {
-			ig.Spec.MaxSize = fi.Int32(2)
-		}
-	}
-
-	if ig.Spec.Image == "" {
-		architecture, err := MachineArchitecture(cloud, ig.Spec.MachineType)
-		if err != nil {
-			return nil, fmt.Errorf("unable to determine machine architecture for InstanceGroup %q: %v", ig.ObjectMeta.Name, err)
-		}
-		ig.Spec.Image = defaultImage(cluster, channel, architecture)
-		if ig.Spec.Image == "" {
-			return nil, fmt.Errorf("unable to determine default image for InstanceGroup %s", ig.ObjectMeta.Name)
-		}
-	}
-
-	if ig.Spec.Tenancy != "" && ig.Spec.Tenancy != "default" {
-		switch cluster.Spec.GetCloudProvider() {
-		case kops.CloudProviderAWS:
-			if _, ok := awsDedicatedInstanceExceptions[ig.Spec.MachineType]; ok {
-				return nil, fmt.Errorf("invalid dedicated instance type: %s", ig.Spec.MachineType)
-			}
-		default:
-			klog.Warning("Trying to set tenancy on non-AWS environment")
-		}
-	}
-
-	if ig.IsMaster() {
-		if len(ig.Spec.Subnets) == 0 {
-			return nil, fmt.Errorf("master InstanceGroup %s did not specify any Subnets", ig.ObjectMeta.Name)
-		}
-	} else if ig.IsAPIServerOnly() && cluster.Spec.IsIPv6Only() {
-		if len(ig.Spec.Subnets) == 0 {
-			for _, subnet := range cluster.Spec.Subnets {
-				if subnet.Type != kops.SubnetTypePrivate && subnet.Type != kops.SubnetTypeUtility {
-					ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name)
-				}
-			}
-		}
-	} else {
-		if len(ig.Spec.Subnets) == 0 {
-			for _, subnet := range cluster.Spec.Subnets {
-				if subnet.Type != kops.SubnetTypeDualStack && subnet.Type != kops.SubnetTypeUtility {
-					ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name)
-				}
-			}
-		}
-
-		if len(ig.Spec.Subnets) == 0 {
-			for _, subnet := range cluster.Spec.Subnets {
-				if subnet.Type != kops.SubnetTypeUtility {
-					ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name)
-				}
-			}
-		}
-	}
-
-	if len(ig.Spec.Subnets) == 0 {
-		return nil, fmt.Errorf("unable to infer any Subnets for InstanceGroup %s ", ig.ObjectMeta.Name)
-	}
-
 	hasGPU := false
 	clusterNvidia := false
 	if cluster.Spec.Containerd != nil && cluster.Spec.Containerd.NvidiaGPU != nil && fi.BoolValue(cluster.Spec.Containerd.NvidiaGPU.Enabled) {
@@ -296,63 +191,3 @@ func defaultMachineType(cloud fi.Cloud, cluster *kops.Cluster, ig *kops.Instance
 	klog.V(2).Infof("Cannot set default MachineType for CloudProvider=%q, Role=%q", cluster.Spec.GetCloudProvider(), ig.Spec.Role)
 	return "", nil
 }
-
-// defaultImage returns the default Image, based on the cloudprovider
-func defaultImage(cluster *kops.Cluster, channel *kops.Channel, architecture architectures.Architecture) string {
-	if channel != nil {
-		var kubernetesVersion *semver.Version
-		if cluster.Spec.KubernetesVersion != "" {
-			var err error
-			kubernetesVersion, err = util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
-			if err != nil {
-				klog.Warningf("cannot parse KubernetesVersion %q in cluster", cluster.Spec.KubernetesVersion)
-			}
-		}
-		if kubernetesVersion != nil {
-			image := channel.FindImage(cluster.Spec.GetCloudProvider(), *kubernetesVersion, architecture)
-			if image != nil {
-				return image.Name
-			}
-		}
-	}
-
-	switch cluster.Spec.GetCloudProvider() {
-	case kops.CloudProviderDO:
-		return defaultDONodeImage
-	}
-	klog.Infof("Cannot set default Image for CloudProvider=%q", cluster.Spec.GetCloudProvider())
-	return ""
-}
-
-func MachineArchitecture(cloud fi.Cloud, machineType string) (architectures.Architecture, error) {
-	if machineType == "" {
-		return architectures.ArchitectureAmd64, nil
-	}
-
-	switch cloud.ProviderID() {
-	case kops.CloudProviderAWS:
-		info, err := cloud.(awsup.AWSCloud).DescribeInstanceType(machineType)
-		if err != nil {
-			return "", fmt.Errorf("error finding instance info for instance type %q: %v", machineType, err)
-		}
-		if info.ProcessorInfo == nil || len(info.ProcessorInfo.SupportedArchitectures) == 0 {
-			return "", fmt.Errorf("error finding architecture info for instance type %q", machineType)
-		}
-		var unsupported []string
-		for _, arch := range info.ProcessorInfo.SupportedArchitectures {
-			// Return the first found supported architecture, in order of popularity
-			switch fi.StringValue(arch) {
-			case ec2.ArchitectureTypeX8664:
-				return architectures.ArchitectureAmd64, nil
-			case ec2.ArchitectureTypeArm64:
-				return architectures.ArchitectureArm64, nil
-			default:
-				unsupported = append(unsupported, fi.StringValue(arch))
-			}
-		}
-		return "", fmt.Errorf("unsupported architecture for instance type %q: %v", machineType, unsupported)
-	default:
-		// No other clouds are known to support any other architectures at this time
-		return architectures.ArchitectureAmd64, nil
-	}
-}

upup/pkg/fi/cloudup/populate_instancegroup_spec_test.go

@@ -70,16 +70,6 @@ func TestPopulateInstanceGroup_Role_Required(t *testing.T) {
 	expectErrorFromPopulateInstanceGroup(t, cluster, g, channel, "spec.role")
 }
 
-func TestPopulateInstanceGroup_Image_Required(t *testing.T) {
-	_, cluster := buildMinimalCluster()
-	g := buildMinimalNodeInstanceGroup()
-	g.Spec.Image = ""
-
-	channel := &kopsapi.Channel{}
-
-	expectErrorFromPopulateInstanceGroup(t, cluster, g, channel, "unable to determine default image for InstanceGroup nodes")
-}
-
 func TestPopulateInstanceGroup_AddTaintsCollision(t *testing.T) {
 	_, cluster := buildMinimalCluster()
 	input := buildMinimalNodeInstanceGroup()