diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index e2aced3a36..b1b35b87bb 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -40,6 +40,7 @@ import ( kopsbase "k8s.io/kops" "k8s.io/kops/cmd/kops/util" + "k8s.io/kops/pkg/apis/kops" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/registry" kopsutil "k8s.io/kops/pkg/apis/kops/util" @@ -68,9 +69,6 @@ type CreateClusterOptions struct { NodeVolumeSize int32 ContainerRuntime string OutDir string - Image string - NodeImage string - MasterImage string DisableSubnetTags bool NetworkCIDR string DNSZone string @@ -549,22 +547,6 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr } } - if c.Image != "" { - for _, group := range instanceGroups { - group.Spec.Image = c.Image - } - } - if c.MasterImage != "" { - for _, group := range masters { - group.Spec.Image = c.MasterImage - } - } - if c.NodeImage != "" { - for _, group := range nodes { - group.Spec.Image = c.NodeImage - } - } - if c.AssociatePublicIP != nil { for _, group := range instanceGroups { group.Spec.AssociatePublicIP = c.AssociatePublicIP @@ -662,19 +644,6 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr return err } - var fullInstanceGroups []*api.InstanceGroup - for _, group := range instanceGroups { - fullGroup, err := cloudup.PopulateInstanceGroupSpec(fullCluster, group, cloud, clusterResult.Channel) - if err != nil { - return err - } - fullGroup.AddInstanceGroupNodeLabel() - if cluster.Spec.GetCloudProvider() == api.CloudProviderGCE { - fullGroup.Spec.NodeLabels["cloud.google.com/metadata-proxy-ready"] = "true" - } - fullInstanceGroups = append(fullInstanceGroups, fullGroup) - } - kubernetesVersion, err := kopsutil.ParseKubernetesVersion(clusterResult.Cluster.Spec.KubernetesVersion) if err != nil { return fmt.Errorf("cannot parse KubernetesVersion %q in cluster: %w", clusterResult.Cluster.Spec.KubernetesVersion, err) @@ -693,16 +662,28 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr addons = append(addons, addon.Objects...) } - err = validation.DeepValidate(fullCluster, fullInstanceGroups, true, nil) - if err != nil { - return err + { + // Build full IG spec to ensure we end up with a valid IG + fullInstanceGroups := []*kops.InstanceGroup{} + for _, group := range instanceGroups { + fullGroup, err := cloudup.PopulateInstanceGroupSpec(cluster, group, cloud, clusterResult.Channel) + if err != nil { + return err + } + fullInstanceGroups = append(fullInstanceGroups, fullGroup) + } + + err = validation.DeepValidate(fullCluster, fullInstanceGroups, true, nil) + if err != nil { + return fmt.Errorf("validation of the full cluster and instance group specs failed: %w", err) + } } if c.DryRun { var obj []runtime.Object obj = append(obj, cluster) - for _, group := range fullInstanceGroups { + for _, group := range instanceGroups { // Cluster name is not populated, and we need it group.ObjectMeta.Labels = make(map[string]string) group.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name @@ -730,7 +711,7 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr } // Note we perform as much validation as we can, before writing a bad config - err = registry.CreateClusterConfig(ctx, clientset, cluster, fullInstanceGroups, addons) + err = registry.CreateClusterConfig(ctx, clientset, cluster, instanceGroups, addons) if err != nil { return fmt.Errorf("error writing updated configuration: %v", err) } diff --git a/pkg/apis/kops/validation/instancegroup_test.go b/pkg/apis/kops/validation/instancegroup_test.go index 84b3e0bb3f..d45e452d8b 100644 --- a/pkg/apis/kops/validation/instancegroup_test.go +++ b/pkg/apis/kops/validation/instancegroup_test.go @@ -385,7 +385,7 @@ func TestIGUpdatePolicy(t *testing.T) { func TestValidInstanceGroup(t *testing.T) { grid := []struct { IG *kops.InstanceGroup - ExpectedErrors int + ExpectedErrors []string Description string }{ { @@ -401,7 +401,7 @@ func TestValidInstanceGroup(t *testing.T) { Image: "my-image", }, }, - ExpectedErrors: 0, + ExpectedErrors: []string{}, Description: "Valid master instance group failed to validate", }, { @@ -417,7 +417,7 @@ func TestValidInstanceGroup(t *testing.T) { Image: "my-image", }, }, - ExpectedErrors: 0, + ExpectedErrors: []string{}, Description: "Valid API Server instance group failed to validate", }, { @@ -433,7 +433,7 @@ func TestValidInstanceGroup(t *testing.T) { Image: "my-image", }, }, - ExpectedErrors: 0, + ExpectedErrors: []string{}, Description: "Valid node instance group failed to validate", }, { @@ -449,13 +449,28 @@ func TestValidInstanceGroup(t *testing.T) { Image: "my-image", }, }, - ExpectedErrors: 0, + ExpectedErrors: []string{}, Description: "Valid bastion instance group failed to validate", }, + { + IG: &kops.InstanceGroup{ + ObjectMeta: v1.ObjectMeta{ + Name: "eu-central-1a", + }, + Spec: kops.InstanceGroupSpec{ + Role: kops.InstanceGroupRoleBastion, + Subnets: []string{"eu-central-1a"}, + MaxSize: fi.Int32(1), + MinSize: fi.Int32(1), + }, + }, + ExpectedErrors: []string{"Forbidden::spec.image"}, + Description: "Valid instance group must have image set", + }, } for _, g := range grid { errList := ValidateInstanceGroup(g.IG, nil, true) - testErrors(t, g.Description, errList, []string{}) + testErrors(t, g.Description, errList, g.ExpectedErrors) } } diff --git a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml index a687fde6e4..942192a433 100644 --- a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml @@ -65,11 +65,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -86,11 +83,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/different-amis/expected-v1alpha2.yaml b/tests/integration/create_cluster/different-amis/expected-v1alpha2.yaml index 99ecbedd4d..ef48c08049 100644 --- a/tests/integration/create_cluster/different-amis/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/different-amis/expected-v1alpha2.yaml @@ -77,11 +77,8 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: apiserver-us-test-1a role: APIServer subnets: - us-test-1a @@ -101,11 +98,8 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.micro - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: bastions role: Bastion subnets: - us-test-1a @@ -125,11 +119,8 @@ spec: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -149,11 +140,8 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml index 6092e69677..8d4dea1fe7 100644 --- a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml @@ -66,12 +66,8 @@ metadata: spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-1 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: master-us-test1-a role: Master subnets: - us-test1 @@ -90,12 +86,8 @@ metadata: spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-2 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: nodes-us-test1-a role: Node subnets: - us-test1 diff --git a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml index 06d987d513..023f487d8a 100644 --- a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml @@ -86,11 +86,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -107,11 +104,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1b role: Master subnets: - us-test-1b @@ -128,11 +122,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1c role: Master subnets: - us-test-1c @@ -149,11 +140,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a @@ -170,11 +158,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1b role: Node subnets: - us-test-1b @@ -191,11 +176,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1c role: Node subnets: - us-test-1c diff --git a/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml index 06d987d513..023f487d8a 100644 --- a/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml @@ -86,11 +86,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -107,11 +104,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1b role: Master subnets: - us-test-1b @@ -128,11 +122,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1c role: Master subnets: - us-test-1c @@ -149,11 +140,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a @@ -170,11 +158,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1b role: Node subnets: - us-test-1b @@ -191,11 +176,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1c role: Node subnets: - us-test-1c diff --git a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml index 26e091c4bc..6c2bc8cc6e 100644 --- a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml @@ -73,12 +73,8 @@ metadata: spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-1 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: master-us-test1-a role: Master subnets: - us-test1 @@ -97,12 +93,8 @@ metadata: spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-1 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: master-us-test1-b role: Master subnets: - us-test1 @@ -121,12 +113,8 @@ metadata: spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-1 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: master-us-test1-c role: Master subnets: - us-test1 @@ -145,12 +133,8 @@ metadata: spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-2 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: nodes-us-test1-a role: Node subnets: - us-test1 @@ -169,12 +153,8 @@ metadata: spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-2 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: nodes-us-test1-b role: Node subnets: - us-test1 @@ -193,12 +173,8 @@ metadata: spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-2 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: nodes-us-test1-c role: Node subnets: - us-test1 diff --git a/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml index 988205cd2f..12ddecb3eb 100644 --- a/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml @@ -78,11 +78,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a-1 role: Master subnets: - us-test-1a @@ -99,11 +96,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a-2 role: Master subnets: - us-test-1a @@ -120,11 +114,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a-3 role: Master subnets: - us-test-1a @@ -141,11 +132,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml index 03e367ebf0..e2d9f88d21 100644 --- a/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml @@ -94,11 +94,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a-1 role: Master subnets: - us-test-1a @@ -115,11 +112,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a-2 role: Master subnets: - us-test-1a @@ -136,11 +130,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a-3 role: Master subnets: - us-test-1a @@ -157,11 +148,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1b-1 role: Master subnets: - us-test-1b @@ -178,11 +166,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1b-2 role: Master subnets: - us-test-1b @@ -199,11 +184,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a @@ -220,11 +202,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1b role: Node subnets: - us-test-1b diff --git a/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml index ccf0524feb..a0e9af691e 100644 --- a/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml @@ -75,11 +75,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.micro - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: bastions role: Bastion subnets: - us-test-1a @@ -96,11 +93,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -117,11 +111,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml b/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml index 072ca088d0..04443b733e 100644 --- a/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml @@ -71,11 +71,8 @@ spec: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -95,11 +92,8 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/karpenter/expected-v1alpha2.yaml b/tests/integration/create_cluster/karpenter/expected-v1alpha2.yaml index c02226a224..18b418704c 100644 --- a/tests/integration/create_cluster/karpenter/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/karpenter/expected-v1alpha2.yaml @@ -83,11 +83,8 @@ spec: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -107,10 +104,6 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required manager: Karpenter - maxSize: 2 - minSize: 2 - nodeLabels: - kops.k8s.io/instancegroup: nodes role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml index eb9f2f7993..50fdf04183 100644 --- a/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml @@ -66,11 +66,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -87,11 +84,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml index eb9f2f7993..50fdf04183 100644 --- a/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml @@ -66,11 +66,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -87,11 +84,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml index 0bf19c0e52..61a810403f 100644 --- a/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml @@ -69,11 +69,8 @@ spec: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -93,11 +90,8 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml index edb33b699f..231ab88fd6 100644 --- a/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml @@ -69,11 +69,8 @@ spec: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -93,11 +90,8 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/minimal-1.24/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.24/expected-v1alpha2.yaml index 2144870ffe..73272d3b9f 100644 --- a/tests/integration/create_cluster/minimal-1.24/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.24/expected-v1alpha2.yaml @@ -69,11 +69,8 @@ spec: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -93,11 +90,8 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/minimal-1.25/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.25/expected-v1alpha2.yaml index f6a9e3cf70..a9c6db44bd 100644 --- a/tests/integration/create_cluster/minimal-1.25/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.25/expected-v1alpha2.yaml @@ -69,11 +69,8 @@ spec: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -93,11 +90,8 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/minimal-1.26/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.26/expected-v1alpha2.yaml index ce037e5f9e..71a3d6ddad 100644 --- a/tests/integration/create_cluster/minimal-1.26/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.26/expected-v1alpha2.yaml @@ -69,11 +69,8 @@ spec: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -93,11 +90,8 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml index 1070babbc7..aa15622709 100644 --- a/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml @@ -75,11 +75,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.micro - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: bastions role: Bastion subnets: - us-test-1a @@ -96,11 +93,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -117,11 +111,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml b/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml index 5cf4057814..9effbee33e 100644 --- a/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml @@ -69,11 +69,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -90,11 +87,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/private/expected-v1alpha2.yaml b/tests/integration/create_cluster/private/expected-v1alpha2.yaml index c9665afdcb..7f87a4c1a2 100644 --- a/tests/integration/create_cluster/private/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private/expected-v1alpha2.yaml @@ -78,11 +78,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.micro - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: bastions role: Bastion subnets: - us-test-1a @@ -102,11 +99,8 @@ spec: - sg-exampleid4 image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -126,11 +120,8 @@ spec: - sg-exampleid2 image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml index 264d20660f..7b40a93655 100644 --- a/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml @@ -73,12 +73,8 @@ metadata: spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: f1-micro - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: bastions role: Bastion subnets: - us-test1 @@ -100,12 +96,8 @@ spec: - sg-exampleid4 image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-1 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: master-us-test1-a role: Master subnets: - us-test1 @@ -127,12 +119,8 @@ spec: - sg-exampleid2 image: ubuntu-os-cloud/ubuntu-2004-focal-v20220118 machineType: n1-standard-2 - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - cloud.google.com/metadata-proxy-ready: "true" - kops.k8s.io/instancegroup: nodes-us-test1-a role: Node subnets: - us-test1 diff --git a/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml index 2cddbe86da..d603cb7e98 100644 --- a/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml @@ -75,11 +75,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -96,11 +93,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml index 34148b4ba7..b2d26d4a9a 100644 --- a/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml @@ -68,11 +68,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -89,11 +86,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml index 34148b4ba7..b2d26d4a9a 100644 --- a/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml @@ -68,11 +68,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -89,11 +86,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml index 0eed390eca..4037c47653 100644 --- a/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml @@ -67,11 +67,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: m3.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: master-us-test-1a role: Master subnets: - us-test-1a @@ -88,11 +85,8 @@ metadata: spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220615 machineType: t2.medium - manager: CloudGroup maxSize: 1 minSize: 1 - nodeLabels: - kops.k8s.io/instancegroup: nodes-us-test-1a role: Node subnets: - us-test-1a diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index be823258bb..3f8ba73d5d 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -819,6 +819,7 @@ func (c *ApplyClusterCmd) upgradeSpecs(assetBuilder *assets.AssetBuilder) error } c.Cluster = fullCluster + klog.Infof("Populating instance group from upgradeSpecs") for i, g := range c.InstanceGroups { fullGroup, err := PopulateInstanceGroupSpec(fullCluster, g, c.Cloud, c.channel) if err != nil { diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index 38d3bb2930..3270103730 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -23,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/blang/semver/v4" "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -30,7 +31,9 @@ import ( "k8s.io/kops" api "k8s.io/kops/pkg/apis/kops" + kopsapi "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/model" + "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/client/simple" "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/featureflag" @@ -40,6 +43,7 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/azure" "k8s.io/kops/upup/pkg/fi/cloudup/gce" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" + "k8s.io/kops/util/pkg/architectures" "k8s.io/kops/util/pkg/vfs" ) @@ -150,6 +154,10 @@ type NewClusterOptions struct { // InstanceManager specifies which manager to use for managing instances. InstanceManager string + + Image string + NodeImage string + MasterImage string } func (o *NewClusterOptions) InitDefaults() { @@ -260,9 +268,18 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster return nil, fmt.Errorf("unable to infer cloud provider from zones. pass in the cloud provider explicitly using --cloud") } } + + var cloud fi.Cloud + switch api.CloudProviderID(opt.CloudProvider) { case api.CloudProviderAWS: cluster.Spec.CloudProvider.AWS = &api.AWSSpec{} + cloudTags := map[string]string{} + awsCloud, err := awsup.NewAWSCloud(opt.Zones[0][:len(opt.Zones[0])-1], cloudTags) + if err != nil { + return nil, err + } + cloud = awsCloud case api.CloudProviderAzure: cluster.Spec.CloudProvider.Azure = &api.AzureSpec{ SubscriptionID: opt.AzureSubscriptionID, @@ -310,7 +327,7 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster } } - err = setupVPC(opt, &cluster) + err = setupVPC(opt, &cluster, cloud) if err != nil { return nil, err } @@ -373,6 +390,96 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster instanceGroups = append(instanceGroups, nodes...) instanceGroups = append(instanceGroups, bastions...) + for _, instanceGroup := range instanceGroups { + g := instanceGroup + ig := g + if instanceGroup.Spec.Image == "" { + if opt.Image != "" { + instanceGroup.Spec.Image = opt.Image + } else { + architecture, err := MachineArchitecture(cloud, instanceGroup.Spec.MachineType) + if err != nil { + return nil, err + } + instanceGroup.Spec.Image = defaultImage(&cluster, channel, architecture) + } + } + + // TODO: Clean up + if g.IsMaster() { + if g.Spec.MachineType == "" { + g.Spec.MachineType, err = defaultMachineType(cloud, &cluster, ig) + if err != nil { + return nil, fmt.Errorf("error assigning default machine type for masters: %v", err) + } + + } + } else if g.Spec.Role == kopsapi.InstanceGroupRoleBastion { + if g.Spec.MachineType == "" { + g.Spec.MachineType, err = defaultMachineType(cloud, &cluster, g) + if err != nil { + return nil, fmt.Errorf("error assigning default machine type for bastions: %v", err) + } + } + } else { + if g.IsAPIServerOnly() && !featureflag.APIServerNodes.Enabled() { + return nil, fmt.Errorf("apiserver nodes requires the APIServerNodes feature flag to be enabled") + } + if g.Spec.MachineType == "" { + g.Spec.MachineType, err = defaultMachineType(cloud, &cluster, g) + if err != nil { + return nil, fmt.Errorf("error assigning default machine type for nodes: %v", err) + } + } + + } + + if ig.Spec.Tenancy != "" && ig.Spec.Tenancy != "default" { + switch cluster.Spec.GetCloudProvider() { + case kopsapi.CloudProviderAWS: + if _, ok := awsDedicatedInstanceExceptions[g.Spec.MachineType]; ok { + return nil, fmt.Errorf("invalid dedicated instance type: %s", g.Spec.MachineType) + } + default: + klog.Warning("Trying to set tenancy on non-AWS environment") + } + } + + if ig.IsMaster() { + if len(ig.Spec.Subnets) == 0 { + return nil, fmt.Errorf("master InstanceGroup %s did not specify any Subnets", g.ObjectMeta.Name) + } + } else if ig.IsAPIServerOnly() && cluster.Spec.IsIPv6Only() { + if len(ig.Spec.Subnets) == 0 { + for _, subnet := range cluster.Spec.Subnets { + if subnet.Type != kopsapi.SubnetTypePrivate && subnet.Type != kopsapi.SubnetTypeUtility { + ig.Spec.Subnets = append(g.Spec.Subnets, subnet.Name) + } + } + } + } else { + if len(ig.Spec.Subnets) == 0 { + for _, subnet := range cluster.Spec.Subnets { + if subnet.Type != kopsapi.SubnetTypeDualStack && subnet.Type != kopsapi.SubnetTypeUtility { + g.Spec.Subnets = append(g.Spec.Subnets, subnet.Name) + } + } + } + + if len(g.Spec.Subnets) == 0 { + for _, subnet := range cluster.Spec.Subnets { + if subnet.Type != kopsapi.SubnetTypeUtility { + g.Spec.Subnets = append(g.Spec.Subnets, subnet.Name) + } + } + } + } + + if len(g.Spec.Subnets) == 0 { + return nil, fmt.Errorf("unable to infer any Subnets for InstanceGroup %s ", g.ObjectMeta.Name) + } + } + result := NewClusterResult{ Cluster: &cluster, InstanceGroups: instanceGroups, @@ -381,17 +488,13 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster return &result, nil } -func setupVPC(opt *NewClusterOptions, cluster *api.Cluster) error { +func setupVPC(opt *NewClusterOptions, cluster *api.Cluster, cloud fi.Cloud) error { cluster.Spec.NetworkID = opt.NetworkID switch cluster.Spec.GetCloudProvider() { case api.CloudProviderAWS: if cluster.Spec.NetworkID == "" && len(opt.SubnetIDs) > 0 { - cloudTags := map[string]string{} - awsCloud, err := awsup.NewAWSCloud(opt.Zones[0][:len(opt.Zones[0])-1], cloudTags) - if err != nil { - return fmt.Errorf("error loading cloud: %v", err) - } + awsCloud := cloud.(awsup.AWSCloud) res, err := awsCloud.EC2().DescribeSubnets(&ec2.DescribeSubnetsInput{ SubnetIds: []*string{aws.String(opt.SubnetIDs[0])}, }) @@ -755,6 +858,8 @@ func setupMasters(opt *NewClusterOptions, cluster *api.Cluster, zoneToSubnetMap } } + g.Spec.Image = opt.MasterImage + masters = append(masters, g) } } @@ -880,6 +985,8 @@ func setupNodes(opt *NewClusterOptions, cluster *api.Cluster, zoneToSubnetMap ma } } + g.Spec.Image = opt.NodeImage + nodes = append(nodes, g) } @@ -1114,6 +1221,8 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S bastionGroup := &api.InstanceGroup{} bastionGroup.Spec.Role = api.InstanceGroupRoleBastion bastionGroup.ObjectMeta.Name = "bastions" + bastionGroup.Spec.MaxSize = fi.Int32(1) + bastionGroup.Spec.MinSize = fi.Int32(1) bastions = append(bastions, bastionGroup) if !dns.IsGossipHostname(cluster.Name) { @@ -1321,3 +1430,63 @@ func addCiliumNetwork(cluster *api.Cluster) { enabled := false cluster.Spec.KubeProxy.Enabled = &enabled } + +// defaultImage returns the default Image, based on the cloudprovider +func defaultImage(cluster *kopsapi.Cluster, channel *kopsapi.Channel, architecture architectures.Architecture) string { + if channel != nil { + var kubernetesVersion *semver.Version + if cluster.Spec.KubernetesVersion != "" { + var err error + kubernetesVersion, err = util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion) + if err != nil { + klog.Warningf("cannot parse KubernetesVersion %q in cluster", cluster.Spec.KubernetesVersion) + } + } + if kubernetesVersion != nil { + image := channel.FindImage(cluster.Spec.GetCloudProvider(), *kubernetesVersion, architecture) + if image != nil { + return image.Name + } + } + } + + switch cluster.Spec.GetCloudProvider() { + case kopsapi.CloudProviderDO: + return defaultDONodeImage + } + klog.Infof("Cannot set default Image for CloudProvider=%q", cluster.Spec.GetCloudProvider()) + return "" +} + +func MachineArchitecture(cloud fi.Cloud, machineType string) (architectures.Architecture, error) { + if machineType == "" { + return architectures.ArchitectureAmd64, nil + } + + switch cloud.ProviderID() { + case kopsapi.CloudProviderAWS: + info, err := cloud.(awsup.AWSCloud).DescribeInstanceType(machineType) + if err != nil { + return "", fmt.Errorf("error finding instance info for instance type %q: %v", machineType, err) + } + if info.ProcessorInfo == nil || len(info.ProcessorInfo.SupportedArchitectures) == 0 { + return "", fmt.Errorf("error finding architecture info for instance type %q", machineType) + } + var unsupported []string + for _, arch := range info.ProcessorInfo.SupportedArchitectures { + // Return the first found supported architecture, in order of popularity + switch fi.StringValue(arch) { + case ec2.ArchitectureTypeX8664: + return architectures.ArchitectureAmd64, nil + case ec2.ArchitectureTypeArm64: + return architectures.ArchitectureArm64, nil + default: + unsupported = append(unsupported, fi.StringValue(arch)) + } + } + return "", fmt.Errorf("unsupported architecture for instance type %q: %v", machineType, unsupported) + default: + // No other clouds are known to support any other architectures at this time + return architectures.ArchitectureAmd64, nil + } +} diff --git a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go index 95f301327d..3c480511c1 100644 --- a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go +++ b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go @@ -20,18 +20,13 @@ import ( "fmt" "strings" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/blang/semver/v4" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/kops/validation" - "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" - "k8s.io/kops/util/pkg/architectures" "k8s.io/kops/util/pkg/reflectutils" ) @@ -67,117 +62,17 @@ var awsDedicatedInstanceExceptions = map[string]bool{ // PopulateInstanceGroupSpec sets default values in the InstanceGroup // The InstanceGroup is simpler than the cluster spec, so we just populate in place (like the rest of k8s) func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup, cloud fi.Cloud, channel *kops.Channel) (*kops.InstanceGroup, error) { + klog.Infof("Populating instance group spec for %q", input.GetName()) + var err error err = validation.ValidateInstanceGroup(input, nil, false).ToAggregate() if err != nil { - return nil, err + return nil, fmt.Errorf("failed validating input specs: %w", err) } ig := &kops.InstanceGroup{} reflectutils.JSONMergeStruct(ig, input) - // TODO: Clean up - if ig.IsMaster() { - if ig.Spec.MachineType == "" { - ig.Spec.MachineType, err = defaultMachineType(cloud, cluster, ig) - if err != nil { - return nil, fmt.Errorf("error assigning default machine type for masters: %v", err) - } - - } - if ig.Spec.MinSize == nil { - ig.Spec.MinSize = fi.Int32(1) - } - if ig.Spec.MaxSize == nil { - ig.Spec.MaxSize = fi.Int32(1) - } - } else if ig.Spec.Role == kops.InstanceGroupRoleBastion { - if ig.Spec.MachineType == "" { - ig.Spec.MachineType, err = defaultMachineType(cloud, cluster, ig) - if err != nil { - return nil, fmt.Errorf("error assigning default machine type for bastions: %v", err) - } - } - if ig.Spec.MinSize == nil { - ig.Spec.MinSize = fi.Int32(1) - } - if ig.Spec.MaxSize == nil { - ig.Spec.MaxSize = fi.Int32(1) - } - } else { - if ig.IsAPIServerOnly() && !featureflag.APIServerNodes.Enabled() { - return nil, fmt.Errorf("apiserver nodes requires the APIServerNodes feature flag to be enabled") - } - if ig.Spec.MachineType == "" { - ig.Spec.MachineType, err = defaultMachineType(cloud, cluster, ig) - if err != nil { - return nil, fmt.Errorf("error assigning default machine type for nodes: %v", err) - } - } - if ig.Spec.MinSize == nil { - ig.Spec.MinSize = fi.Int32(2) - } - if ig.Spec.MaxSize == nil { - ig.Spec.MaxSize = fi.Int32(2) - } - } - - if ig.Spec.Image == "" { - architecture, err := MachineArchitecture(cloud, ig.Spec.MachineType) - if err != nil { - return nil, fmt.Errorf("unable to determine machine architecture for InstanceGroup %q: %v", ig.ObjectMeta.Name, err) - } - ig.Spec.Image = defaultImage(cluster, channel, architecture) - if ig.Spec.Image == "" { - return nil, fmt.Errorf("unable to determine default image for InstanceGroup %s", ig.ObjectMeta.Name) - } - } - - if ig.Spec.Tenancy != "" && ig.Spec.Tenancy != "default" { - switch cluster.Spec.GetCloudProvider() { - case kops.CloudProviderAWS: - if _, ok := awsDedicatedInstanceExceptions[ig.Spec.MachineType]; ok { - return nil, fmt.Errorf("invalid dedicated instance type: %s", ig.Spec.MachineType) - } - default: - klog.Warning("Trying to set tenancy on non-AWS environment") - } - } - - if ig.IsMaster() { - if len(ig.Spec.Subnets) == 0 { - return nil, fmt.Errorf("master InstanceGroup %s did not specify any Subnets", ig.ObjectMeta.Name) - } - } else if ig.IsAPIServerOnly() && cluster.Spec.IsIPv6Only() { - if len(ig.Spec.Subnets) == 0 { - for _, subnet := range cluster.Spec.Subnets { - if subnet.Type != kops.SubnetTypePrivate && subnet.Type != kops.SubnetTypeUtility { - ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name) - } - } - } - } else { - if len(ig.Spec.Subnets) == 0 { - for _, subnet := range cluster.Spec.Subnets { - if subnet.Type != kops.SubnetTypeDualStack && subnet.Type != kops.SubnetTypeUtility { - ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name) - } - } - } - - if len(ig.Spec.Subnets) == 0 { - for _, subnet := range cluster.Spec.Subnets { - if subnet.Type != kops.SubnetTypeUtility { - ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name) - } - } - } - } - - if len(ig.Spec.Subnets) == 0 { - return nil, fmt.Errorf("unable to infer any Subnets for InstanceGroup %s ", ig.ObjectMeta.Name) - } - hasGPU := false clusterNvidia := false if cluster.Spec.Containerd != nil && cluster.Spec.Containerd.NvidiaGPU != nil && fi.BoolValue(cluster.Spec.Containerd.NvidiaGPU.Enabled) { @@ -296,63 +191,3 @@ func defaultMachineType(cloud fi.Cloud, cluster *kops.Cluster, ig *kops.Instance klog.V(2).Infof("Cannot set default MachineType for CloudProvider=%q, Role=%q", cluster.Spec.GetCloudProvider(), ig.Spec.Role) return "", nil } - -// defaultImage returns the default Image, based on the cloudprovider -func defaultImage(cluster *kops.Cluster, channel *kops.Channel, architecture architectures.Architecture) string { - if channel != nil { - var kubernetesVersion *semver.Version - if cluster.Spec.KubernetesVersion != "" { - var err error - kubernetesVersion, err = util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion) - if err != nil { - klog.Warningf("cannot parse KubernetesVersion %q in cluster", cluster.Spec.KubernetesVersion) - } - } - if kubernetesVersion != nil { - image := channel.FindImage(cluster.Spec.GetCloudProvider(), *kubernetesVersion, architecture) - if image != nil { - return image.Name - } - } - } - - switch cluster.Spec.GetCloudProvider() { - case kops.CloudProviderDO: - return defaultDONodeImage - } - klog.Infof("Cannot set default Image for CloudProvider=%q", cluster.Spec.GetCloudProvider()) - return "" -} - -func MachineArchitecture(cloud fi.Cloud, machineType string) (architectures.Architecture, error) { - if machineType == "" { - return architectures.ArchitectureAmd64, nil - } - - switch cloud.ProviderID() { - case kops.CloudProviderAWS: - info, err := cloud.(awsup.AWSCloud).DescribeInstanceType(machineType) - if err != nil { - return "", fmt.Errorf("error finding instance info for instance type %q: %v", machineType, err) - } - if info.ProcessorInfo == nil || len(info.ProcessorInfo.SupportedArchitectures) == 0 { - return "", fmt.Errorf("error finding architecture info for instance type %q", machineType) - } - var unsupported []string - for _, arch := range info.ProcessorInfo.SupportedArchitectures { - // Return the first found supported architecture, in order of popularity - switch fi.StringValue(arch) { - case ec2.ArchitectureTypeX8664: - return architectures.ArchitectureAmd64, nil - case ec2.ArchitectureTypeArm64: - return architectures.ArchitectureArm64, nil - default: - unsupported = append(unsupported, fi.StringValue(arch)) - } - } - return "", fmt.Errorf("unsupported architecture for instance type %q: %v", machineType, unsupported) - default: - // No other clouds are known to support any other architectures at this time - return architectures.ArchitectureAmd64, nil - } -} diff --git a/upup/pkg/fi/cloudup/populate_instancegroup_spec_test.go b/upup/pkg/fi/cloudup/populate_instancegroup_spec_test.go index 7ca1bc946c..a713b19dd9 100644 --- a/upup/pkg/fi/cloudup/populate_instancegroup_spec_test.go +++ b/upup/pkg/fi/cloudup/populate_instancegroup_spec_test.go @@ -70,16 +70,6 @@ func TestPopulateInstanceGroup_Role_Required(t *testing.T) { expectErrorFromPopulateInstanceGroup(t, cluster, g, channel, "spec.role") } -func TestPopulateInstanceGroup_Image_Required(t *testing.T) { - _, cluster := buildMinimalCluster() - g := buildMinimalNodeInstanceGroup() - g.Spec.Image = "" - - channel := &kopsapi.Channel{} - - expectErrorFromPopulateInstanceGroup(t, cluster, g, channel, "unable to determine default image for InstanceGroup nodes") -} - func TestPopulateInstanceGroup_AddTaintsCollision(t *testing.T) { _, cluster := buildMinimalCluster() input := buildMinimalNodeInstanceGroup()