kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make kops get instances work with karpenter

This commit is contained in:
Ole Markus With 2021-12-06 20:27:25 +01:00
parent 043dd91417
commit d4ec7cc7e0
31 changed files with 214 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/model/awsmodel/autoscalinggroup.go
View File

@ -24,6 +24,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/klog/v2"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/featureflag"
"k8s.io/kops/pkg/model"
@ -79,12 +80,14 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
c.AddTask(task)
// @step: now lets build the autoscaling group task
tsk, err := b.buildAutoScalingGroupTask(c, name, ig)
if err != nil {
return err
if ig.Spec.InstanceManager != "Karpenter" {
tsk, err := b.buildAutoScalingGroupTask(c, name, ig)
if err != nil {
return err
}
tsk.LaunchTemplate = task
c.AddTask(tsk)
}
tsk.LaunchTemplate = task
c.AddTask(tsk)
warmPool := b.Cluster.Spec.WarmPool.ResolveDefaults(ig)
{

9
pkg/model/awsmodel/nodeterminationhandler.go
View File

@ -25,6 +25,7 @@ import (
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"github.com/aws/aws-sdk-go/aws"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/upup/pkg/fi"
)
@ -80,9 +81,11 @@ type NodeTerminationHandlerBuilder struct {
func (b *NodeTerminationHandlerBuilder) Build(c *fi.ModelBuilderContext) error {
for _, ig := range b.InstanceGroups {
err := b.configureASG(c, ig)
if err != nil {
return err
if ig.Spec.InstanceManager == kops.InstanceManagerCloudGroup {
err := b.configureASG(c, ig)
if err != nil {
return err
}
}
}

2
tests/integration/create_cluster/complex/expected-v1alpha2.yaml
View File

@ -64,6 +64,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -84,6 +85,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml
View File

@ -65,6 +65,7 @@ metadata:
name: master-us-test1-a
spec:
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-1
maxSize: 1
minSize: 1
@ -88,6 +89,7 @@ metadata:
name: nodes-us-test1-a
spec:
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-2
maxSize: 1
minSize: 1

6
tests/integration/create_cluster/ha/expected-v1alpha2.yaml
View File

@ -85,6 +85,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -105,6 +106,7 @@ metadata:
name: master-us-test-1b
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -125,6 +127,7 @@ metadata:
name: master-us-test-1c
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -145,6 +148,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1
@ -165,6 +169,7 @@ metadata:
name: nodes-us-test-1b
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1
@ -185,6 +190,7 @@ metadata:
name: nodes-us-test-1c
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

6
tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml
View File

@ -85,6 +85,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -105,6 +106,7 @@ metadata:
name: master-us-test-1b
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -125,6 +127,7 @@ metadata:
name: master-us-test-1c
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -145,6 +148,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1
@ -165,6 +169,7 @@ metadata:
name: nodes-us-test-1b
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1
@ -185,6 +190,7 @@ metadata:
name: nodes-us-test-1c
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

6
tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml
View File

@ -73,6 +73,7 @@ metadata:
name: master-us-test1-a
spec:
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-1
maxSize: 1
minSize: 1
@ -96,6 +97,7 @@ metadata:
name: master-us-test1-b
spec:
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-1
maxSize: 1
minSize: 1
@ -119,6 +121,7 @@ metadata:
name: master-us-test1-c
spec:
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-1
maxSize: 1
minSize: 1
@ -142,6 +145,7 @@ metadata:
name: nodes-us-test1-a
spec:
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-2
maxSize: 1
minSize: 1
@ -165,6 +169,7 @@ metadata:
name: nodes-us-test1-b
spec:
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-2
maxSize: 1
minSize: 1
@ -188,6 +193,7 @@ metadata:
name: nodes-us-test1-c
spec:
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-2
maxSize: 1
minSize: 1

4
tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml
View File

@ -77,6 +77,7 @@ metadata:
name: master-us-test-1a-1
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -97,6 +98,7 @@ metadata:
name: master-us-test-1a-2
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -117,6 +119,7 @@ metadata:
name: master-us-test-1a-3
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -137,6 +140,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

7
tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml
View File

@ -93,6 +93,7 @@ metadata:
name: master-us-test-1a-1
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -113,6 +114,7 @@ metadata:
name: master-us-test-1a-2
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -133,6 +135,7 @@ metadata:
name: master-us-test-1a-3
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -153,6 +156,7 @@ metadata:
name: master-us-test-1b-1
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -173,6 +177,7 @@ metadata:
name: master-us-test-1b-2
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -193,6 +198,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1
@ -213,6 +219,7 @@ metadata:
name: nodes-us-test-1b
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

3
tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml
View File

@ -74,6 +74,7 @@ metadata:
name: bastions
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.micro
maxSize: 1
minSize: 1
@ -94,6 +95,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -114,6 +116,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml
View File

@ -67,6 +67,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
instanceMetadata:
httpPutResponseHopLimit: 3
httpTokens: required
@ -90,6 +91,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
instanceMetadata:
httpPutResponseHopLimit: 1
httpTokens: required

2
tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml
View File

@ -65,6 +65,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -85,6 +86,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml
View File

@ -65,6 +65,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -85,6 +86,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml
View File

@ -65,6 +65,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -85,6 +86,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml
View File

@ -65,6 +65,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -85,6 +86,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml
View File

@ -65,6 +65,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
instanceMetadata:
httpPutResponseHopLimit: 3
httpTokens: required
@ -88,6 +89,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
instanceMetadata:
httpPutResponseHopLimit: 1
httpTokens: required

2
tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml
View File

@ -65,6 +65,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
instanceMetadata:
httpPutResponseHopLimit: 3
httpTokens: required
@ -88,6 +89,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
instanceMetadata:
httpPutResponseHopLimit: 1
httpTokens: required

3
tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml
View File

@ -74,6 +74,7 @@ metadata:
name: bastions
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.micro
maxSize: 1
minSize: 1
@ -94,6 +95,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -114,6 +116,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/overrides/expected-v1alpha2.yaml
View File

@ -68,6 +68,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -88,6 +89,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

3
tests/integration/create_cluster/private/expected-v1alpha2.yaml
View File

@ -77,6 +77,7 @@ metadata:
name: bastions
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.micro
maxSize: 1
minSize: 1
@ -100,6 +101,7 @@ spec:
- sg-exampleid3
- sg-exampleid4
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -123,6 +125,7 @@ spec:
- sg-exampleid
- sg-exampleid2
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

3
tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml
View File

@ -72,6 +72,7 @@ metadata:
name: bastions
spec:
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: f1-micro
maxSize: 1
minSize: 1
@ -98,6 +99,7 @@ spec:
- sg-exampleid3
- sg-exampleid4
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-1
maxSize: 1
minSize: 1
@ -124,6 +126,7 @@ spec:
- sg-exampleid
- sg-exampleid2
image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118
instanceManager: CloudGroup
machineType: n1-standard-2
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml
View File

@ -74,6 +74,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -94,6 +95,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml
View File

@ -67,6 +67,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -87,6 +88,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml
View File

@ -67,6 +67,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -87,6 +88,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml
View File

@ -66,6 +66,7 @@ metadata:
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: m3.medium
maxSize: 1
minSize: 1
@ -86,6 +87,7 @@ metadata:
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118
instanceManager: CloudGroup
machineType: t2.medium
maxSize: 1
minSize: 1

2
tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content
View File

@ -54,7 +54,7 @@ spec:
version: 9.99.0
- id: k8s-1.19
manifest: karpenter.sh/k8s-1.19.yaml
manifestHash: d31ab5979f05095a9ec57bbbf1a2ac14a52a9e321d7732832dcd14f44be39551
manifestHash: f5bf5ee51335406c3ad93bfa6cea5778dfddba76f353da2b89bfdc30b925bd13
name: karpenter.sh
needsPKI: true
selector:

2
tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content
View File

@ -864,7 +864,7 @@ spec:
cpu: 1000
provider:
instanceProfile: nodes.minimal.example.com
launchTemplate: karpenter-nodes
launchTemplate: karpenter-nodes.minimal.example.com
securityGroupSelector:
Name: nodes.minimal.example.com
subnetSelector:

53
tests/integration/update_cluster/karpenter/kubernetes.tf
View File

@ -10,7 +10,7 @@ locals {
master_security_group_ids = [aws_security_group.masters-minimal-example-com.id]
masters_role_arn = aws_iam_role.masters-minimal-example-com.arn
masters_role_name = aws_iam_role.masters-minimal-example-com.name
node_autoscaling_group_ids = [aws_autoscaling_group.karpenter-nodes-minimal-example-com.id, aws_autoscaling_group.nodes-minimal-example-com.id]
node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id]
node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id]
node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id]
nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn
@ -67,7 +67,7 @@ output "masters_role_name" {
}
output "node_autoscaling_group_ids" {
value = [aws_autoscaling_group.karpenter-nodes-minimal-example-com.id, aws_autoscaling_group.nodes-minimal-example-com.id]
value = [aws_autoscaling_group.nodes-minimal-example-com.id]
}
output "node_security_group_ids" {
@ -115,55 +115,6 @@ provider "aws" {
region = "us-test-1"
}
resource "aws_autoscaling_group" "karpenter-nodes-minimal-example-com" {
enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
launch_template {
id = aws_launch_template.karpenter-nodes-minimal-example-com.id
version = aws_launch_template.karpenter-nodes-minimal-example-com.latest_version
}
max_size = 2
metrics_granularity = "1Minute"
min_size = 2
name = "karpenter-nodes.minimal.example.com"
protect_from_scale_in = false
tag {
key = "KubernetesCluster"
propagate_at_launch = true
value = "minimal.example.com"
}
tag {
key = "Name"
propagate_at_launch = true
value = "karpenter-nodes.minimal.example.com"
}
tag {
key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
propagate_at_launch = true
value = "node"
}
tag {
key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
propagate_at_launch = true
value = ""
}
tag {
key = "k8s.io/role/node"
propagate_at_launch = true
value = "1"
}
tag {
key = "kops.k8s.io/instancegroup"
propagate_at_launch = true
value = "karpenter-nodes"
}
tag {
key = "kubernetes.io/cluster/minimal.example.com"
propagate_at_launch = true
value = "owned"
}
vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id]
}
resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" {
enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
launch_template {

2
upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template
View File

@ -667,7 +667,7 @@ spec:
cpu: 1000
provider:
instanceProfile: nodes.{{ ClusterName }}
launchTemplate: {{ $name }}
launchTemplate: {{ $name }}.{{ ClusterName }}
securityGroupSelector:
Name: nodes.{{ ClusterName }}
subnetSelector:

123
upup/pkg/fi/cloudup/awsup/aws_cloud.go
View File

@ -51,6 +51,8 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
k8s_aws "k8s.io/legacy-cloud-providers/aws"
"k8s.io/kops/dnsprovider/pkg/dnsprovider"
dnsproviderroute53 "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/aws/route53"
"k8s.io/kops/pkg/apis/kops"
@ -60,7 +62,6 @@ import (
identity_aws "k8s.io/kops/pkg/nodeidentity/aws"
"k8s.io/kops/pkg/resources/spotinst"
"k8s.io/kops/upup/pkg/fi"
k8s_aws "k8s.io/legacy-cloud-providers/aws"
)
// By default, aws-sdk-go only retries 3 times, which doesn't give
@ -622,7 +623,125 @@ func (c *awsCloudImplementation) GetCloudGroups(cluster *kops.Cluster, instanceg
return sgroups, nil
}
return getCloudGroups(c, cluster, instancegroups, warnUnmatched, nodes)
cloudGroups, err := getCloudGroups(c, cluster, instancegroups, warnUnmatched, nodes)
if err != nil {
return nil, err
}
karpenterGroups, err := getKarpenterGroups(c, cluster, instancegroups, nodes)
if err != nil {
return nil, err
}
for name, group := range karpenterGroups {
cloudGroups[name] = group
}
return cloudGroups, nil
}
func getKarpenterGroups(c AWSCloud, cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) {
cloudGroups := make(map[string]*cloudinstances.CloudInstanceGroup)
for _, ig := range instancegroups {
if ig.Spec.InstanceManager == kops.InstanceManagerKarpenter {
group, err := buildKarpenterGroup(c, cluster, ig, nodes)
if err != nil {
return nil, err
}
cloudGroups[ig.ObjectMeta.Name] = group
}
}
return cloudGroups, nil
}
func buildKarpenterGroup(c AWSCloud, cluster *kops.Cluster, ig *kops.InstanceGroup, nodes []v1.Node) (*cloudinstances.CloudInstanceGroup, error) {
nodeMap := cloudinstances.GetNodeMap(nodes, cluster)
instances := make(map[string]*ec2.Instance)
updatedInstances := make(map[string]*ec2.Instance)
clusterName := c.Tags()[TagClusterName]
var version string
{
result, err := c.EC2().DescribeLaunchTemplates(&ec2.DescribeLaunchTemplatesInput{
Filters: []*ec2.Filter{
NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name),
NewEC2Filter("tag:"+TagClusterName, clusterName),
},
})
if err != nil {
return nil, err
}
lt := result.LaunchTemplates[0]
versionNumber := *lt.LatestVersionNumber
version = strconv.Itoa(int(versionNumber))
}
karpenterGroup := &cloudinstances.CloudInstanceGroup{
InstanceGroup: ig,
HumanName: ig.ObjectMeta.Name,
}
{
req := &ec2.DescribeInstancesInput{
Filters: []*ec2.Filter{
NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name),
NewEC2Filter("tag:"+TagClusterName, clusterName),
NewEC2Filter("instance-state-name", "pending", "running", "stopping", "stopped"),
},
}
result, err := c.EC2().DescribeInstances(req)
if err != nil {
return nil, err
}
for _, r := range result.Reservations {
for _, i := range r.Instances {
id := aws.StringValue(i.InstanceId)
instances[id] = i
}
}
}
klog.Infof("found %d karpenter instances", len(instances))
{
req := &ec2.DescribeInstancesInput{
Filters: []*ec2.Filter{
NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name),
NewEC2Filter("tag:"+TagClusterName, clusterName),
NewEC2Filter("instance-state-name", "pending", "running", "stopping", "stopped"),
NewEC2Filter("tag:aws:ec2launchtemplate:version", version),
},
}
result, err := c.EC2().DescribeInstances(req)
if err != nil {
return nil, err
}
for _, r := range result.Reservations {
for _, i := range r.Instances {
id := aws.StringValue(i.InstanceId)
updatedInstances[id] = i
}
}
}
klog.Infof("found %d updated instances", len(updatedInstances))
{
for _, instance := range instances {
id := *instance.InstanceId
_, ready := updatedInstances[id]
var status string
if ready {
status = cloudinstances.CloudInstanceStatusUpToDate
} else {
status = cloudinstances.CloudInstanceStatusNeedsUpdate
}
cloudInstance, _ := karpenterGroup.NewCloudInstance(id, status, nodeMap[id])
addCloudInstanceData(cloudInstance, instance)
}
}
return karpenterGroup, nil
}
func getCloudGroups(c AWSCloud, cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, warnUnmatched bool, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) {

5
upup/pkg/fi/cloudup/populate_instancegroup_spec.go
View File

@ -22,6 +22,7 @@ import (
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/blang/semver/v4"
"k8s.io/klog/v2"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/kops/util"
"k8s.io/kops/pkg/apis/kops/validation"
@ -173,6 +174,10 @@ func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup,
}
}
}
if ig.Spec.InstanceManager == "" {
ig.Spec.InstanceManager = kops.InstanceManagerCloudGroup
}
return ig, nil
}