diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 394e2d2e9b..3de993608e 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "k8s.io/klog/v2" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model" @@ -79,12 +80,14 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(task) // @step: now lets build the autoscaling group task - tsk, err := b.buildAutoScalingGroupTask(c, name, ig) - if err != nil { - return err + if ig.Spec.InstanceManager != "Karpenter" { + tsk, err := b.buildAutoScalingGroupTask(c, name, ig) + if err != nil { + return err + } + tsk.LaunchTemplate = task + c.AddTask(tsk) } - tsk.LaunchTemplate = task - c.AddTask(tsk) warmPool := b.Cluster.Spec.WarmPool.ResolveDefaults(ig) { diff --git a/pkg/model/awsmodel/nodeterminationhandler.go b/pkg/model/awsmodel/nodeterminationhandler.go index 772413e2fa..3287e2dc9e 100644 --- a/pkg/model/awsmodel/nodeterminationhandler.go +++ b/pkg/model/awsmodel/nodeterminationhandler.go @@ -25,6 +25,7 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "github.com/aws/aws-sdk-go/aws" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" ) @@ -80,9 +81,11 @@ type NodeTerminationHandlerBuilder struct { func (b *NodeTerminationHandlerBuilder) Build(c *fi.ModelBuilderContext) error { for _, ig := range b.InstanceGroups { - err := b.configureASG(c, ig) - if err != nil { - return err + if ig.Spec.InstanceManager == kops.InstanceManagerCloudGroup { + err := b.configureASG(c, ig) + if err != nil { + return err + } } } diff --git a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml index 1b416599ca..ecf6f14ac7 100644 --- a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml @@ -64,6 +64,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -84,6 +85,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml index d433f0a5be..bf0f00317f 100644 --- a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -88,6 +89,7 @@ metadata: name: nodes-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml index a2131e0256..4142d2e51d 100644 --- a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml @@ -85,6 +85,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -105,6 +106,7 @@ metadata: name: master-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -125,6 +127,7 @@ metadata: name: master-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -145,6 +148,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -165,6 +169,7 @@ metadata: name: nodes-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -185,6 +190,7 @@ metadata: name: nodes-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml index a2131e0256..4142d2e51d 100644 --- a/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml @@ -85,6 +85,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -105,6 +106,7 @@ metadata: name: master-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -125,6 +127,7 @@ metadata: name: master-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -145,6 +148,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -165,6 +169,7 @@ metadata: name: nodes-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -185,6 +190,7 @@ metadata: name: nodes-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml index e1ab6da893..22cc833ed4 100644 --- a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml @@ -73,6 +73,7 @@ metadata: name: master-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -96,6 +97,7 @@ metadata: name: master-us-test1-b spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -119,6 +121,7 @@ metadata: name: master-us-test1-c spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -142,6 +145,7 @@ metadata: name: nodes-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 @@ -165,6 +169,7 @@ metadata: name: nodes-us-test1-b spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 @@ -188,6 +193,7 @@ metadata: name: nodes-us-test1-c spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml index 1e10778abf..d1f94b6bc0 100644 --- a/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml @@ -77,6 +77,7 @@ metadata: name: master-us-test-1a-1 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -97,6 +98,7 @@ metadata: name: master-us-test-1a-2 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -117,6 +119,7 @@ metadata: name: master-us-test-1a-3 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -137,6 +140,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml index 411454465c..16da87cbc0 100644 --- a/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml @@ -93,6 +93,7 @@ metadata: name: master-us-test-1a-1 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -113,6 +114,7 @@ metadata: name: master-us-test-1a-2 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -133,6 +135,7 @@ metadata: name: master-us-test-1a-3 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -153,6 +156,7 @@ metadata: name: master-us-test-1b-1 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -173,6 +177,7 @@ metadata: name: master-us-test-1b-2 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -193,6 +198,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -213,6 +219,7 @@ metadata: name: nodes-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml index 729838a4f8..8e7178a6a7 100644 --- a/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml @@ -74,6 +74,7 @@ metadata: name: bastions spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.micro maxSize: 1 minSize: 1 @@ -94,6 +95,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -114,6 +116,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml b/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml index b62c91b66e..5304026462 100644 --- a/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml @@ -67,6 +67,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required @@ -90,6 +91,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 1 httpTokens: required diff --git a/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml index f8a2a4e58c..ec4a6fb0ee 100644 --- a/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -85,6 +86,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml index 8aa10625ee..4cec7f0f24 100644 --- a/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -85,6 +86,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml index bbb082cb66..3960ae47c9 100644 --- a/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -85,6 +86,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml index bbb082cb66..3960ae47c9 100644 --- a/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -85,6 +86,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml index 60ece11212..7dde515496 100644 --- a/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required @@ -88,6 +89,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 1 httpTokens: required diff --git a/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml index 67d0345559..2462129ec2 100644 --- a/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required @@ -88,6 +89,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 1 httpTokens: required diff --git a/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml index b432f78124..0a798de54d 100644 --- a/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml @@ -74,6 +74,7 @@ metadata: name: bastions spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.micro maxSize: 1 minSize: 1 @@ -94,6 +95,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -114,6 +116,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml b/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml index a0a1174099..7f435ad173 100644 --- a/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml @@ -68,6 +68,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -88,6 +89,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/private/expected-v1alpha2.yaml b/tests/integration/create_cluster/private/expected-v1alpha2.yaml index 9d5e89339c..693f50ecdc 100644 --- a/tests/integration/create_cluster/private/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private/expected-v1alpha2.yaml @@ -77,6 +77,7 @@ metadata: name: bastions spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.micro maxSize: 1 minSize: 1 @@ -100,6 +101,7 @@ spec: - sg-exampleid3 - sg-exampleid4 image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -123,6 +125,7 @@ spec: - sg-exampleid - sg-exampleid2 image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml index 4c8f859085..59aa216768 100644 --- a/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml @@ -72,6 +72,7 @@ metadata: name: bastions spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: f1-micro maxSize: 1 minSize: 1 @@ -98,6 +99,7 @@ spec: - sg-exampleid3 - sg-exampleid4 image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -124,6 +126,7 @@ spec: - sg-exampleid - sg-exampleid2 image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml index 10ca073d3c..6e70791362 100644 --- a/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml @@ -74,6 +74,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -94,6 +95,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml index a2847cf5bc..36357e5a5f 100644 --- a/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml @@ -67,6 +67,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -87,6 +88,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml index a2847cf5bc..36357e5a5f 100644 --- a/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml @@ -67,6 +67,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -87,6 +88,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml index 86dafd4d05..b00d62cefa 100644 --- a/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml @@ -66,6 +66,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -86,6 +87,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index 18d504a801..a3798692b0 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -54,7 +54,7 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: d31ab5979f05095a9ec57bbbf1a2ac14a52a9e321d7732832dcd14f44be39551 + manifestHash: f5bf5ee51335406c3ad93bfa6cea5778dfddba76f353da2b89bfdc30b925bd13 name: karpenter.sh needsPKI: true selector: diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index ece199ae8c..e45fa59d72 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -864,7 +864,7 @@ spec: cpu: 1000 provider: instanceProfile: nodes.minimal.example.com - launchTemplate: karpenter-nodes + launchTemplate: karpenter-nodes.minimal.example.com securityGroupSelector: Name: nodes.minimal.example.com subnetSelector: diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index 1e923988c7..6e38358a48 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -10,7 +10,7 @@ locals { master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] masters_role_arn = aws_iam_role.masters-minimal-example-com.arn masters_role_name = aws_iam_role.masters-minimal-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.karpenter-nodes-minimal-example-com.id, aws_autoscaling_group.nodes-minimal-example-com.id] + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn @@ -67,7 +67,7 @@ output "masters_role_name" { } output "node_autoscaling_group_ids" { - value = [aws_autoscaling_group.karpenter-nodes-minimal-example-com.id, aws_autoscaling_group.nodes-minimal-example-com.id] + value = [aws_autoscaling_group.nodes-minimal-example-com.id] } output "node_security_group_ids" { @@ -115,55 +115,6 @@ provider "aws" { region = "us-test-1" } -resource "aws_autoscaling_group" "karpenter-nodes-minimal-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.karpenter-nodes-minimal-example-com.id - version = aws_launch_template.karpenter-nodes-minimal-example-com.latest_version - } - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 - name = "karpenter-nodes.minimal.example.com" - protect_from_scale_in = false - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "minimal.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "karpenter-nodes.minimal.example.com" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "node" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/role/node" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "karpenter-nodes" - } - tag { - key = "kubernetes.io/cluster/minimal.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] -} - resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" { enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] launch_template { diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 151a235840..26f2605958 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -667,7 +667,7 @@ spec: cpu: 1000 provider: instanceProfile: nodes.{{ ClusterName }} - launchTemplate: {{ $name }} + launchTemplate: {{ $name }}.{{ ClusterName }} securityGroupSelector: Name: nodes.{{ ClusterName }} subnetSelector: diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index df96ea5d69..072d5d571e 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -51,6 +51,8 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + k8s_aws "k8s.io/legacy-cloud-providers/aws" + "k8s.io/kops/dnsprovider/pkg/dnsprovider" dnsproviderroute53 "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/aws/route53" "k8s.io/kops/pkg/apis/kops" @@ -60,7 +62,6 @@ import ( identity_aws "k8s.io/kops/pkg/nodeidentity/aws" "k8s.io/kops/pkg/resources/spotinst" "k8s.io/kops/upup/pkg/fi" - k8s_aws "k8s.io/legacy-cloud-providers/aws" ) // By default, aws-sdk-go only retries 3 times, which doesn't give @@ -622,7 +623,125 @@ func (c *awsCloudImplementation) GetCloudGroups(cluster *kops.Cluster, instanceg return sgroups, nil } - return getCloudGroups(c, cluster, instancegroups, warnUnmatched, nodes) + cloudGroups, err := getCloudGroups(c, cluster, instancegroups, warnUnmatched, nodes) + if err != nil { + return nil, err + } + karpenterGroups, err := getKarpenterGroups(c, cluster, instancegroups, nodes) + if err != nil { + return nil, err + } + + for name, group := range karpenterGroups { + cloudGroups[name] = group + } + return cloudGroups, nil +} + +func getKarpenterGroups(c AWSCloud, cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) { + cloudGroups := make(map[string]*cloudinstances.CloudInstanceGroup) + for _, ig := range instancegroups { + if ig.Spec.InstanceManager == kops.InstanceManagerKarpenter { + group, err := buildKarpenterGroup(c, cluster, ig, nodes) + if err != nil { + return nil, err + } + cloudGroups[ig.ObjectMeta.Name] = group + } + } + return cloudGroups, nil +} + +func buildKarpenterGroup(c AWSCloud, cluster *kops.Cluster, ig *kops.InstanceGroup, nodes []v1.Node) (*cloudinstances.CloudInstanceGroup, error) { + nodeMap := cloudinstances.GetNodeMap(nodes, cluster) + instances := make(map[string]*ec2.Instance) + updatedInstances := make(map[string]*ec2.Instance) + clusterName := c.Tags()[TagClusterName] + var version string + + { + result, err := c.EC2().DescribeLaunchTemplates(&ec2.DescribeLaunchTemplatesInput{ + Filters: []*ec2.Filter{ + NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name), + NewEC2Filter("tag:"+TagClusterName, clusterName), + }, + }) + if err != nil { + return nil, err + } + lt := result.LaunchTemplates[0] + versionNumber := *lt.LatestVersionNumber + version = strconv.Itoa(int(versionNumber)) + + } + + karpenterGroup := &cloudinstances.CloudInstanceGroup{ + InstanceGroup: ig, + HumanName: ig.ObjectMeta.Name, + } + { + req := &ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name), + NewEC2Filter("tag:"+TagClusterName, clusterName), + NewEC2Filter("instance-state-name", "pending", "running", "stopping", "stopped"), + }, + } + + result, err := c.EC2().DescribeInstances(req) + if err != nil { + return nil, err + } + + for _, r := range result.Reservations { + for _, i := range r.Instances { + id := aws.StringValue(i.InstanceId) + instances[id] = i + } + } + } + + klog.Infof("found %d karpenter instances", len(instances)) + + { + req := &ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name), + NewEC2Filter("tag:"+TagClusterName, clusterName), + NewEC2Filter("instance-state-name", "pending", "running", "stopping", "stopped"), + NewEC2Filter("tag:aws:ec2launchtemplate:version", version), + }, + } + + result, err := c.EC2().DescribeInstances(req) + if err != nil { + return nil, err + } + + for _, r := range result.Reservations { + for _, i := range r.Instances { + id := aws.StringValue(i.InstanceId) + updatedInstances[id] = i + } + } + } + klog.Infof("found %d updated instances", len(updatedInstances)) + + { + for _, instance := range instances { + id := *instance.InstanceId + _, ready := updatedInstances[id] + var status string + if ready { + status = cloudinstances.CloudInstanceStatusUpToDate + } else { + status = cloudinstances.CloudInstanceStatusNeedsUpdate + } + cloudInstance, _ := karpenterGroup.NewCloudInstance(id, status, nodeMap[id]) + addCloudInstanceData(cloudInstance, instance) + } + } + return karpenterGroup, nil } func getCloudGroups(c AWSCloud, cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, warnUnmatched bool, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) { diff --git a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go index 1f9d241d48..d817ca7744 100644 --- a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go +++ b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go @@ -22,6 +22,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/blang/semver/v4" "k8s.io/klog/v2" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/kops/validation" @@ -173,6 +174,10 @@ func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup, } } } + + if ig.Spec.InstanceManager == "" { + ig.Spec.InstanceManager = kops.InstanceManagerCloudGroup + } return ig, nil }