diff --git a/docs/operations/karpenter.md b/docs/operations/karpenter.md index 89e1bd0870..38714aef73 100644 --- a/docs/operations/karpenter.md +++ b/docs/operations/karpenter.md @@ -45,16 +45,16 @@ A Karpenter-managed InstanceGroup controls a corresponding Karpenter Provisioner Note that not all features of InstanceGroups are supported. +## Subnets + +By default, kOps will tag subnets with `kops.k8s.io/instance-group/: "true"` for each InstanceGroup the subnet is assigned to. If you enable manual tagging of subnets, you have to ensure these tags are added, if not Karpenter will fail to provision any instances. + ## Instance Types If you do not specify a mixed instances policy, only the instance type specified by `spec.machineType` will be used. With Karpenter, one typically wants a wider range of instances to choose from. kOps supports both providing a list of instance types through `spec.mixedInstancesPolicy.instances` and providing instance type requirements through `spec.mixedInstancesPolicy.instanceRequirements`. See (/instance_groups)[InstanceGroup documentation] for more details. ## Known limitations -### Subnet selections - -kOps will ignore the InstanceGroup `spec.subnets` configuration and unconditionally add all subnets eligible to run Nodes to all Provisioners. It is not currently possible to create Provisioners limited only to certain subnets. - ### Karpenter-managed Launch Templates On EKS, Karpener creates its own launch templates for Provisioners. These launch templates will not work with a kOps cluster for a number of reasons. Most importantly, they do not use supported AMIs and they do not install and configure nodeup, the instance-side kOps component. The Karpenter features that require Karpenter to directly manage launch templates will not be available on kOps. diff --git a/pkg/model/awsmodel/network.go b/pkg/model/awsmodel/network.go index a00a58bb34..a3f7b515f2 100644 --- a/pkg/model/awsmodel/network.go +++ b/pkg/model/awsmodel/network.go @@ -256,6 +256,15 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { default: klog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) } + + for _, ig := range b.InstanceGroups { + for _, igSubnetName := range ig.Spec.Subnets { + if subnetSpec.Name == igSubnetName { + tags["kops.k8s.io/instance-group/"+ig.GetName()] = "true" + } + } + } + } else { klog.V(2).Infof("skipping subnet tags. Ensure these are maintained externally.") } diff --git a/tests/integration/update_cluster/apiservernodes/cloudformation.json b/tests/integration/update_cluster/apiservernodes/cloudformation.json index f2ef3cfa25..a8cbd71fd2 100644 --- a/tests/integration/update_cluster/apiservernodes/cloudformation.json +++ b/tests/integration/update_cluster/apiservernodes/cloudformation.json @@ -992,6 +992,18 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/apiserver", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/apiservernodes/kubernetes.tf b/tests/integration/update_cluster/apiservernodes/kubernetes.tf index e0d0c2e20c..0888548354 100644 --- a/tests/integration/update_cluster/apiservernodes/kubernetes.tf +++ b/tests/integration/update_cluster/apiservernodes/kubernetes.tf @@ -962,12 +962,15 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/apiserver" = "true" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/aws-lb-controller/kubernetes.tf b/tests/integration/update_cluster/aws-lb-controller/kubernetes.tf index c5d6909d69..790a41986a 100644 --- a/tests/integration/update_cluster/aws-lb-controller/kubernetes.tf +++ b/tests/integration/update_cluster/aws-lb-controller/kubernetes.tf @@ -876,12 +876,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf index 2483c933d3..78def7b5cd 100644 --- a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf +++ b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf @@ -1188,6 +1188,8 @@ resource "aws_subnet" "us-test-1a-bastionuserdata-example-com" { "KubernetesCluster" = "bastionuserdata.example.com" "Name" = "us-test-1a.bastionuserdata.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/bastionuserdata.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1201,6 +1203,7 @@ resource "aws_subnet" "utility-us-test-1a-bastionuserdata-example-com" { "KubernetesCluster" = "bastionuserdata.example.com" "Name" = "utility-us-test-1a.bastionuserdata.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/bastionuserdata.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/complex/cloudformation.json b/tests/integration/update_cluster/complex/cloudformation.json index 1ab2d661f5..357eedd0a2 100644 --- a/tests/integration/update_cluster/complex/cloudformation.json +++ b/tests/integration/update_cluster/complex/cloudformation.json @@ -1224,6 +1224,14 @@ "Key": "foo/bar", "Value": "fib+baz" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/complex.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/complex/kubernetes.tf b/tests/integration/update_cluster/complex/kubernetes.tf index d5e4bf9a43..5b53bbc5f7 100644 --- a/tests/integration/update_cluster/complex/kubernetes.tf +++ b/tests/integration/update_cluster/complex/kubernetes.tf @@ -1149,14 +1149,16 @@ resource "aws_subnet" "us-test-1a-complex-example-com" { enable_resource_name_dns_a_record_on_launch = true private_dns_hostname_type_on_launch = "resource-name" tags = { - "KubernetesCluster" = "complex.example.com" - "Name" = "us-test-1a.complex.example.com" - "Owner" = "John Doe" - "SubnetType" = "Public" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/complex.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "complex.example.com" + "Name" = "us-test-1a.complex.example.com" + "Owner" = "John Doe" + "SubnetType" = "Public" + "foo/bar" = "fib+baz" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/complex.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.complex-example-com.id } diff --git a/tests/integration/update_cluster/compress/kubernetes.tf b/tests/integration/update_cluster/compress/kubernetes.tf index 8053f1f60c..2d3aa38149 100644 --- a/tests/integration/update_cluster/compress/kubernetes.tf +++ b/tests/integration/update_cluster/compress/kubernetes.tf @@ -757,12 +757,14 @@ resource "aws_subnet" "us-test-1a-compress-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "compress.example.com" - "Name" = "us-test-1a.compress.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/compress.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "compress.example.com" + "Name" = "us-test-1a.compress.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/compress.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.compress-example-com.id } diff --git a/tests/integration/update_cluster/containerd-custom/cloudformation.json b/tests/integration/update_cluster/containerd-custom/cloudformation.json index b1338ad060..5abe0a843f 100644 --- a/tests/integration/update_cluster/containerd-custom/cloudformation.json +++ b/tests/integration/update_cluster/containerd-custom/cloudformation.json @@ -785,6 +785,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/containerd.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/containerd/cloudformation.json b/tests/integration/update_cluster/containerd/cloudformation.json index b1338ad060..5abe0a843f 100644 --- a/tests/integration/update_cluster/containerd/cloudformation.json +++ b/tests/integration/update_cluster/containerd/cloudformation.json @@ -785,6 +785,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/containerd.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/digit/kubernetes.tf b/tests/integration/update_cluster/digit/kubernetes.tf index 24178834a4..cdcd898b3a 100644 --- a/tests/integration/update_cluster/digit/kubernetes.tf +++ b/tests/integration/update_cluster/digit/kubernetes.tf @@ -859,12 +859,14 @@ resource "aws_subnet" "us-test-1a-123-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "123.example.com" - "Name" = "us-test-1a.123.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/123.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "123.example.com" + "Name" = "us-test-1a.123.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/123.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.prefix_123-example-com.id } diff --git a/tests/integration/update_cluster/docker-custom/cloudformation.json b/tests/integration/update_cluster/docker-custom/cloudformation.json index bef4732539..aaa06a9720 100644 --- a/tests/integration/update_cluster/docker-custom/cloudformation.json +++ b/tests/integration/update_cluster/docker-custom/cloudformation.json @@ -785,6 +785,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/docker.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/existing_iam/kubernetes.tf b/tests/integration/update_cluster/existing_iam/kubernetes.tf index 4c0fb2135f..629a24f3a1 100644 --- a/tests/integration/update_cluster/existing_iam/kubernetes.tf +++ b/tests/integration/update_cluster/existing_iam/kubernetes.tf @@ -1102,6 +1102,8 @@ resource "aws_subnet" "us-test-1a-existing-iam-example-com" { "KubernetesCluster" = "existing-iam.example.com" "Name" = "us-test-1a.existing-iam.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/existing-iam.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" @@ -1116,6 +1118,7 @@ resource "aws_subnet" "us-test-1b-existing-iam-example-com" { "KubernetesCluster" = "existing-iam.example.com" "Name" = "us-test-1b.existing-iam.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1b" = "true" "kubernetes.io/cluster/existing-iam.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" @@ -1130,6 +1133,7 @@ resource "aws_subnet" "us-test-1c-existing-iam-example-com" { "KubernetesCluster" = "existing-iam.example.com" "Name" = "us-test-1c.existing-iam.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1c" = "true" "kubernetes.io/cluster/existing-iam.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/existing_iam_cloudformation/cloudformation.json b/tests/integration/update_cluster/existing_iam_cloudformation/cloudformation.json index 82f0e41649..7c36b5091e 100644 --- a/tests/integration/update_cluster/existing_iam_cloudformation/cloudformation.json +++ b/tests/integration/update_cluster/existing_iam_cloudformation/cloudformation.json @@ -781,6 +781,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/existing_sg/kubernetes.tf b/tests/integration/update_cluster/existing_sg/kubernetes.tf index 6fcd5b3910..3b17bdf391 100644 --- a/tests/integration/update_cluster/existing_sg/kubernetes.tf +++ b/tests/integration/update_cluster/existing_sg/kubernetes.tf @@ -1475,6 +1475,8 @@ resource "aws_subnet" "us-test-1a-existingsg-example-com" { "KubernetesCluster" = "existingsg.example.com" "Name" = "us-test-1a.existingsg.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/existingsg.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" @@ -1489,6 +1491,7 @@ resource "aws_subnet" "us-test-1b-existingsg-example-com" { "KubernetesCluster" = "existingsg.example.com" "Name" = "us-test-1b.existingsg.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1b" = "true" "kubernetes.io/cluster/existingsg.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" @@ -1503,6 +1506,7 @@ resource "aws_subnet" "us-test-1c-existingsg-example-com" { "KubernetesCluster" = "existingsg.example.com" "Name" = "us-test-1c.existingsg.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1c" = "true" "kubernetes.io/cluster/existingsg.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/external_dns/cloudformation.json b/tests/integration/update_cluster/external_dns/cloudformation.json index 3bdefab92a..918f11bb35 100644 --- a/tests/integration/update_cluster/external_dns/cloudformation.json +++ b/tests/integration/update_cluster/external_dns/cloudformation.json @@ -785,6 +785,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/external_dns/kubernetes.tf b/tests/integration/update_cluster/external_dns/kubernetes.tf index a33607b580..bc6bc03c94 100644 --- a/tests/integration/update_cluster/external_dns/kubernetes.tf +++ b/tests/integration/update_cluster/external_dns/kubernetes.tf @@ -769,12 +769,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf b/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf index 6438e161e6..86adf0bfad 100644 --- a/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf +++ b/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf @@ -834,12 +834,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/externallb/cloudformation.json b/tests/integration/update_cluster/externallb/cloudformation.json index d35b7758ea..102cd394db 100644 --- a/tests/integration/update_cluster/externallb/cloudformation.json +++ b/tests/integration/update_cluster/externallb/cloudformation.json @@ -801,6 +801,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/externallb.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/externallb/kubernetes.tf b/tests/integration/update_cluster/externallb/kubernetes.tf index ed56d099d2..1b8c165b48 100644 --- a/tests/integration/update_cluster/externallb/kubernetes.tf +++ b/tests/integration/update_cluster/externallb/kubernetes.tf @@ -776,6 +776,8 @@ resource "aws_subnet" "us-test-1a-externallb-example-com" { "KubernetesCluster" = "externallb.example.com" "Name" = "us-test-1a.externallb.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/externallb.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/externalpolicies/kubernetes.tf b/tests/integration/update_cluster/externalpolicies/kubernetes.tf index bc79501794..691423efb4 100644 --- a/tests/integration/update_cluster/externalpolicies/kubernetes.tf +++ b/tests/integration/update_cluster/externalpolicies/kubernetes.tf @@ -966,6 +966,8 @@ resource "aws_subnet" "us-test-1a-externalpolicies-example-com" { "Owner" = "John Doe" "SubnetType" = "Public" "foo/bar" = "fib+baz" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/externalpolicies.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/ha/kubernetes.tf b/tests/integration/update_cluster/ha/kubernetes.tf index 8409703cd5..c702d9e4cf 100644 --- a/tests/integration/update_cluster/ha/kubernetes.tf +++ b/tests/integration/update_cluster/ha/kubernetes.tf @@ -1171,12 +1171,14 @@ resource "aws_subnet" "us-test-1a-ha-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "ha.example.com" - "Name" = "us-test-1a.ha.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/ha.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "ha.example.com" + "Name" = "us-test-1a.ha.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/ha.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.ha-example-com.id } @@ -1185,12 +1187,14 @@ resource "aws_subnet" "us-test-1b-ha-example-com" { availability_zone = "us-test-1b" cidr_block = "172.20.64.0/19" tags = { - "KubernetesCluster" = "ha.example.com" - "Name" = "us-test-1b.ha.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/ha.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "ha.example.com" + "Name" = "us-test-1b.ha.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1b" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/ha.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.ha-example-com.id } @@ -1199,12 +1203,14 @@ resource "aws_subnet" "us-test-1c-ha-example-com" { availability_zone = "us-test-1c" cidr_block = "172.20.96.0/19" tags = { - "KubernetesCluster" = "ha.example.com" - "Name" = "us-test-1c.ha.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/ha.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "ha.example.com" + "Name" = "us-test-1c.ha.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1c" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/ha.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.ha-example-com.id } diff --git a/tests/integration/update_cluster/irsa/kubernetes.tf b/tests/integration/update_cluster/irsa/kubernetes.tf index d7d04be341..9d470be3b4 100644 --- a/tests/integration/update_cluster/irsa/kubernetes.tf +++ b/tests/integration/update_cluster/irsa/kubernetes.tf @@ -900,12 +900,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index b09ad03f11..c109401e77 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -48,7 +48,7 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: cd6c83c893a2bfb1543fee16c9ae5ebeede469f41556a7830bb50697e4ff7942 + manifestHash: 6c7b7d9130ccb04026fa2aa28aa8a318e9882fa15ebd4c1f3b03f5a468a647f8 name: karpenter.sh selector: k8s-addon: karpenter.sh diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index 5be66a0796..a6c6458902 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -1359,8 +1359,8 @@ spec: provider: launchTemplate: karpenter-nodes-default.minimal.example.com subnetSelector: + kops.k8s.io/instance-group/karpenter-nodes-default: '*' kubernetes.io/cluster/minimal.example.com: '*' - kubernetes.io/role/internal-elb: "1" requirements: - key: karpenter.sh/capacity-type operator: In @@ -1391,8 +1391,8 @@ spec: provider: launchTemplate: karpenter-nodes-single-machinetype.minimal.example.com subnetSelector: + kops.k8s.io/instance-group/karpenter-nodes-single-machinetype: '*' kubernetes.io/cluster/minimal.example.com: '*' - kubernetes.io/role/internal-elb: "1" requirements: - key: karpenter.sh/capacity-type operator: In diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index e2a810553e..30cf12f5ad 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -1034,12 +1034,16 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/karpenter-nodes-default" = "true" + "kops.k8s.io/instance-group/karpenter-nodes-single-machinetype" = "true" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/lifecycle_phases/network-kubernetes.tf b/tests/integration/update_cluster/lifecycle_phases/network-kubernetes.tf index 2062f57756..33f3272ce9 100644 --- a/tests/integration/update_cluster/lifecycle_phases/network-kubernetes.tf +++ b/tests/integration/update_cluster/lifecycle_phases/network-kubernetes.tf @@ -133,6 +133,8 @@ resource "aws_subnet" "us-test-1a-lifecyclephases-example-com" { "KubernetesCluster" = "lifecyclephases.example.com" "Name" = "us-test-1a.lifecyclephases.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/lifecyclephases.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -146,6 +148,7 @@ resource "aws_subnet" "utility-us-test-1a-lifecyclephases-example-com" { "KubernetesCluster" = "lifecyclephases.example.com" "Name" = "utility-us-test-1a.lifecyclephases.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/lifecyclephases.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa/kubernetes.tf b/tests/integration/update_cluster/many-addons-ccm-irsa/kubernetes.tf index 309c107e0c..817e509337 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-ccm-irsa/kubernetes.tf @@ -1037,12 +1037,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa23/kubernetes.tf b/tests/integration/update_cluster/many-addons-ccm-irsa23/kubernetes.tf index b2474ff657..97d4b5f526 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa23/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-ccm-irsa23/kubernetes.tf @@ -1047,12 +1047,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { enable_resource_name_dns_a_record_on_launch = true private_dns_hostname_type_on_launch = "resource-name" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa24/kubernetes.tf b/tests/integration/update_cluster/many-addons-ccm-irsa24/kubernetes.tf index 2d554df728..f127e23645 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa24/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/kubernetes.tf @@ -1023,12 +1023,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { enable_resource_name_dns_a_record_on_launch = true private_dns_hostname_type_on_launch = "resource-name" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/many-addons-ccm/kubernetes.tf b/tests/integration/update_cluster/many-addons-ccm/kubernetes.tf index 920b979e8d..f7754c4081 100644 --- a/tests/integration/update_cluster/many-addons-ccm/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-ccm/kubernetes.tf @@ -842,12 +842,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/many-addons/kubernetes.tf b/tests/integration/update_cluster/many-addons/kubernetes.tf index cab123a5db..7013d77662 100644 --- a/tests/integration/update_cluster/many-addons/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons/kubernetes.tf @@ -834,12 +834,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/minimal-1.23/kubernetes.tf b/tests/integration/update_cluster/minimal-1.23/kubernetes.tf index 991a7dfd20..edbe55e73f 100644 --- a/tests/integration/update_cluster/minimal-1.23/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-1.23/kubernetes.tf @@ -828,12 +828,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/minimal-1.24/kubernetes.tf b/tests/integration/update_cluster/minimal-1.24/kubernetes.tf index dd0d4a2a52..f7a4e50fa5 100644 --- a/tests/integration/update_cluster/minimal-1.24/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-1.24/kubernetes.tf @@ -814,12 +814,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { enable_resource_name_dns_a_record_on_launch = true private_dns_hostname_type_on_launch = "resource-name" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/minimal-etcd/cloudformation.json b/tests/integration/update_cluster/minimal-etcd/cloudformation.json index df4974719b..353d99f4d8 100644 --- a/tests/integration/update_cluster/minimal-etcd/cloudformation.json +++ b/tests/integration/update_cluster/minimal-etcd/cloudformation.json @@ -785,6 +785,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal-etcd.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/minimal-gp3/cloudformation.json b/tests/integration/update_cluster/minimal-gp3/cloudformation.json index f66d644060..2200cf076e 100644 --- a/tests/integration/update_cluster/minimal-gp3/cloudformation.json +++ b/tests/integration/update_cluster/minimal-gp3/cloudformation.json @@ -781,6 +781,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/minimal-gp3/kubernetes.tf b/tests/integration/update_cluster/minimal-gp3/kubernetes.tf index e298f4f952..32d20a53e6 100644 --- a/tests/integration/update_cluster/minimal-gp3/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-gp3/kubernetes.tf @@ -765,12 +765,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/minimal-ipv6-calico/cloudformation.json b/tests/integration/update_cluster/minimal-ipv6-calico/cloudformation.json index 66da6103e6..45d9ded0da 100644 --- a/tests/integration/update_cluster/minimal-ipv6-calico/cloudformation.json +++ b/tests/integration/update_cluster/minimal-ipv6-calico/cloudformation.json @@ -994,6 +994,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal-ipv6.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/minimal-ipv6-calico/kubernetes.tf b/tests/integration/update_cluster/minimal-ipv6-calico/kubernetes.tf index 9bdee79dd7..522cec23fa 100644 --- a/tests/integration/update_cluster/minimal-ipv6-calico/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-ipv6-calico/kubernetes.tf @@ -987,6 +987,8 @@ resource "aws_subnet" "us-test-1a-minimal-ipv6-example-com" { "KubernetesCluster" = "minimal-ipv6.example.com" "Name" = "us-test-1a.minimal-ipv6.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/minimal-ipv6.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/minimal-ipv6-cilium/cloudformation.json b/tests/integration/update_cluster/minimal-ipv6-cilium/cloudformation.json index bec3fa491c..71b69977e5 100644 --- a/tests/integration/update_cluster/minimal-ipv6-cilium/cloudformation.json +++ b/tests/integration/update_cluster/minimal-ipv6-cilium/cloudformation.json @@ -997,6 +997,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal-ipv6.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/minimal-ipv6-cilium/kubernetes.tf b/tests/integration/update_cluster/minimal-ipv6-cilium/kubernetes.tf index 0288221f7b..a5a4133ac5 100644 --- a/tests/integration/update_cluster/minimal-ipv6-cilium/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-ipv6-cilium/kubernetes.tf @@ -979,6 +979,8 @@ resource "aws_subnet" "us-test-1a-minimal-ipv6-example-com" { "KubernetesCluster" = "minimal-ipv6.example.com" "Name" = "us-test-1a.minimal-ipv6.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/minimal-ipv6.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf b/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf index 75490125c9..12f3834d92 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf @@ -1096,6 +1096,7 @@ resource "aws_subnet" "dualstack-us-test-1a-minimal-ipv6-example-com" { "KubernetesCluster" = "minimal-ipv6.example.com" "Name" = "dualstack-us-test-1a.minimal-ipv6.example.com" "SubnetType" = "DualStack" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" "kubernetes.io/cluster/minimal-ipv6.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1130,6 +1131,7 @@ resource "aws_subnet" "us-test-1a-minimal-ipv6-example-com" { "KubernetesCluster" = "minimal-ipv6.example.com" "Name" = "us-test-1a.minimal-ipv6.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/minimal-ipv6.example.com" = "owned" } vpc_id = aws_vpc.minimal-ipv6-example-com.id @@ -1146,6 +1148,7 @@ resource "aws_subnet" "us-test-1b-minimal-ipv6-example-com" { "KubernetesCluster" = "minimal-ipv6.example.com" "Name" = "us-test-1b.minimal-ipv6.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/minimal-ipv6.example.com" = "owned" } vpc_id = aws_vpc.minimal-ipv6-example-com.id diff --git a/tests/integration/update_cluster/minimal-ipv6/cloudformation.json b/tests/integration/update_cluster/minimal-ipv6/cloudformation.json index bec3fa491c..71b69977e5 100644 --- a/tests/integration/update_cluster/minimal-ipv6/cloudformation.json +++ b/tests/integration/update_cluster/minimal-ipv6/cloudformation.json @@ -997,6 +997,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal-ipv6.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/minimal-ipv6/kubernetes.tf b/tests/integration/update_cluster/minimal-ipv6/kubernetes.tf index 2524c29565..fa87be89fe 100644 --- a/tests/integration/update_cluster/minimal-ipv6/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-ipv6/kubernetes.tf @@ -971,6 +971,8 @@ resource "aws_subnet" "us-test-1a-minimal-ipv6-example-com" { "KubernetesCluster" = "minimal-ipv6.example.com" "Name" = "us-test-1a.minimal-ipv6.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/minimal-ipv6.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/minimal-longclustername/cloudformation.json b/tests/integration/update_cluster/minimal-longclustername/cloudformation.json index 26a8491330..1ab07b1ac9 100644 --- a/tests/integration/update_cluster/minimal-longclustername/cloudformation.json +++ b/tests/integration/update_cluster/minimal-longclustername/cloudformation.json @@ -785,6 +785,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/this.is.truly.a.really.really.long.cluster-name.minimal.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/minimal-longclustername/kubernetes.tf b/tests/integration/update_cluster/minimal-longclustername/kubernetes.tf index 2bdc3251e4..43b7c3a006 100644 --- a/tests/integration/update_cluster/minimal-longclustername/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-longclustername/kubernetes.tf @@ -772,6 +772,8 @@ resource "aws_subnet" "us-test-1a-this-is-truly-a-really-really-long-cluster-nam "KubernetesCluster" = "this.is.truly.a.really.really.long.cluster-name.minimal.example.com" "Name" = "us-test-1a.this.is.truly.a.really.really.long.cluster-name.minimal.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/this.is.truly.a.really.really.long.cluster-name.minimal.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/minimal-warmpool/kubernetes.tf b/tests/integration/update_cluster/minimal-warmpool/kubernetes.tf index 11f4a2152b..cdb6aae447 100644 --- a/tests/integration/update_cluster/minimal-warmpool/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-warmpool/kubernetes.tf @@ -796,6 +796,8 @@ resource "aws_subnet" "us-test-1a-minimal-warmpool-example-com" { "KubernetesCluster" = "minimal-warmpool.example.com" "Name" = "us-test-1a.minimal-warmpool.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/minimal-warmpool.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/minimal/cloudformation.json b/tests/integration/update_cluster/minimal/cloudformation.json index 3bdefab92a..918f11bb35 100644 --- a/tests/integration/update_cluster/minimal/cloudformation.json +++ b/tests/integration/update_cluster/minimal/cloudformation.json @@ -785,6 +785,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/minimal/kubernetes.tf b/tests/integration/update_cluster/minimal/kubernetes.tf index e2d72862cd..949699c1fb 100644 --- a/tests/integration/update_cluster/minimal/kubernetes.tf +++ b/tests/integration/update_cluster/minimal/kubernetes.tf @@ -769,12 +769,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/minimal_gossip/kubernetes.tf b/tests/integration/update_cluster/minimal_gossip/kubernetes.tf index 0200f0d0f8..fc6ae76cab 100644 --- a/tests/integration/update_cluster/minimal_gossip/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gossip/kubernetes.tf @@ -769,12 +769,14 @@ resource "aws_subnet" "us-test-1a-minimal-k8s-local" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.k8s.local" - "Name" = "us-test-1a.minimal.k8s.local" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.k8s.local" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.k8s.local" + "Name" = "us-test-1a.minimal.k8s.local" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.k8s.local" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-k8s-local.id } diff --git a/tests/integration/update_cluster/minimal_gossip_irsa/kubernetes.tf b/tests/integration/update_cluster/minimal_gossip_irsa/kubernetes.tf index b27fb9bc93..1bf1e73e83 100644 --- a/tests/integration/update_cluster/minimal_gossip_irsa/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gossip_irsa/kubernetes.tf @@ -808,12 +808,14 @@ resource "aws_subnet" "us-test-1a-minimal-k8s-local" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.k8s.local" - "Name" = "us-test-1a.minimal.k8s.local" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.k8s.local" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.k8s.local" + "Name" = "us-test-1a.minimal.k8s.local" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.k8s.local" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-k8s-local.id } diff --git a/tests/integration/update_cluster/mixed_instances/cloudformation.json b/tests/integration/update_cluster/mixed_instances/cloudformation.json index 0423c8ceb1..04f2ec6e67 100644 --- a/tests/integration/update_cluster/mixed_instances/cloudformation.json +++ b/tests/integration/update_cluster/mixed_instances/cloudformation.json @@ -1301,6 +1301,10 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/mixedinstances.example.com", "Value": "owned" @@ -1337,6 +1341,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1b", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/mixedinstances.example.com", "Value": "owned" @@ -1373,6 +1385,10 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1c", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/mixedinstances.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/mixed_instances/kubernetes.tf b/tests/integration/update_cluster/mixed_instances/kubernetes.tf index add3ae15f4..c18e304af3 100644 --- a/tests/integration/update_cluster/mixed_instances/kubernetes.tf +++ b/tests/integration/update_cluster/mixed_instances/kubernetes.tf @@ -1192,6 +1192,7 @@ resource "aws_subnet" "us-test-1a-mixedinstances-example-com" { "KubernetesCluster" = "mixedinstances.example.com" "Name" = "us-test-1a.mixedinstances.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" "kubernetes.io/cluster/mixedinstances.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" @@ -1206,6 +1207,8 @@ resource "aws_subnet" "us-test-1b-mixedinstances-example-com" { "KubernetesCluster" = "mixedinstances.example.com" "Name" = "us-test-1b.mixedinstances.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1b" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/mixedinstances.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" @@ -1220,6 +1223,7 @@ resource "aws_subnet" "us-test-1c-mixedinstances-example-com" { "KubernetesCluster" = "mixedinstances.example.com" "Name" = "us-test-1c.mixedinstances.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1c" = "true" "kubernetes.io/cluster/mixedinstances.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/mixed_instances_spot/cloudformation.json b/tests/integration/update_cluster/mixed_instances_spot/cloudformation.json index 8918e0e0cb..4967365aba 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/cloudformation.json +++ b/tests/integration/update_cluster/mixed_instances_spot/cloudformation.json @@ -1301,6 +1301,10 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/mixedinstances.example.com", "Value": "owned" @@ -1337,6 +1341,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1b", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/mixedinstances.example.com", "Value": "owned" @@ -1373,6 +1385,10 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1c", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/mixedinstances.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/mixed_instances_spot/kubernetes.tf b/tests/integration/update_cluster/mixed_instances_spot/kubernetes.tf index b6aed877cd..ace7204ec4 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/kubernetes.tf +++ b/tests/integration/update_cluster/mixed_instances_spot/kubernetes.tf @@ -1192,6 +1192,7 @@ resource "aws_subnet" "us-test-1a-mixedinstances-example-com" { "KubernetesCluster" = "mixedinstances.example.com" "Name" = "us-test-1a.mixedinstances.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" "kubernetes.io/cluster/mixedinstances.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" @@ -1206,6 +1207,8 @@ resource "aws_subnet" "us-test-1b-mixedinstances-example-com" { "KubernetesCluster" = "mixedinstances.example.com" "Name" = "us-test-1b.mixedinstances.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1b" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/mixedinstances.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" @@ -1220,6 +1223,7 @@ resource "aws_subnet" "us-test-1c-mixedinstances-example-com" { "KubernetesCluster" = "mixedinstances.example.com" "Name" = "us-test-1c.mixedinstances.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1c" = "true" "kubernetes.io/cluster/mixedinstances.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json index 3b062e8737..cffc884aac 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json +++ b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json @@ -835,6 +835,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/nthsqsresources.longclustername.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf b/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf index d6969b08c6..6134eeb814 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf +++ b/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf @@ -898,6 +898,8 @@ resource "aws_subnet" "us-test-1a-nthsqsresources-longclustername-example-com" { "KubernetesCluster" = "nthsqsresources.longclustername.example.com" "Name" = "us-test-1a.nthsqsresources.longclustername.example.com" "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/nthsqsresources.longclustername.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/tests/integration/update_cluster/nvidia/cloudformation.json b/tests/integration/update_cluster/nvidia/cloudformation.json index d7005ecc99..1b90bcb0ff 100644 --- a/tests/integration/update_cluster/nvidia/cloudformation.json +++ b/tests/integration/update_cluster/nvidia/cloudformation.json @@ -798,6 +798,14 @@ "Key": "SubnetType", "Value": "Public" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/minimal.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/nvidia/kubernetes.tf b/tests/integration/update_cluster/nvidia/kubernetes.tf index 08e4889eb3..0d5bdeb0a8 100644 --- a/tests/integration/update_cluster/nvidia/kubernetes.tf +++ b/tests/integration/update_cluster/nvidia/kubernetes.tf @@ -785,12 +785,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/private-shared-ip/cloudformation.json b/tests/integration/update_cluster/private-shared-ip/cloudformation.json index 47ec40812d..d29162ce12 100644 --- a/tests/integration/update_cluster/private-shared-ip/cloudformation.json +++ b/tests/integration/update_cluster/private-shared-ip/cloudformation.json @@ -1199,6 +1199,14 @@ "Key": "SubnetType", "Value": "Private" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/private-shared-ip.example.com", "Value": "owned" @@ -1229,6 +1237,10 @@ "Key": "SubnetType", "Value": "Utility" }, + { + "Key": "kops.k8s.io/instance-group/bastion", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/private-shared-ip.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/private-shared-ip/kubernetes.tf b/tests/integration/update_cluster/private-shared-ip/kubernetes.tf index c250c1ec0c..f3b12c1975 100644 --- a/tests/integration/update_cluster/private-shared-ip/kubernetes.tf +++ b/tests/integration/update_cluster/private-shared-ip/kubernetes.tf @@ -1156,6 +1156,8 @@ resource "aws_subnet" "us-test-1a-private-shared-ip-example-com" { "KubernetesCluster" = "private-shared-ip.example.com" "Name" = "us-test-1a.private-shared-ip.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/private-shared-ip.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1169,6 +1171,7 @@ resource "aws_subnet" "utility-us-test-1a-private-shared-ip-example-com" { "KubernetesCluster" = "private-shared-ip.example.com" "Name" = "utility-us-test-1a.private-shared-ip.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/private-shared-ip.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privatecalico/cloudformation.json b/tests/integration/update_cluster/privatecalico/cloudformation.json index ffa2ddf927..c650e785fd 100644 --- a/tests/integration/update_cluster/privatecalico/cloudformation.json +++ b/tests/integration/update_cluster/privatecalico/cloudformation.json @@ -1280,6 +1280,14 @@ "Key": "SubnetType", "Value": "Private" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/privatecalico.example.com", "Value": "owned" @@ -1312,6 +1320,10 @@ "Key": "SubnetType", "Value": "Utility" }, + { + "Key": "kops.k8s.io/instance-group/bastion", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/privatecalico.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/privatecalico/kubernetes.tf b/tests/integration/update_cluster/privatecalico/kubernetes.tf index 8d55c104f1..630559f8e9 100644 --- a/tests/integration/update_cluster/privatecalico/kubernetes.tf +++ b/tests/integration/update_cluster/privatecalico/kubernetes.tf @@ -1213,6 +1213,8 @@ resource "aws_subnet" "us-test-1a-privatecalico-example-com" { "KubernetesCluster" = "privatecalico.example.com" "Name" = "us-test-1a.privatecalico.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privatecalico.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1228,6 +1230,7 @@ resource "aws_subnet" "utility-us-test-1a-privatecalico-example-com" { "KubernetesCluster" = "privatecalico.example.com" "Name" = "utility-us-test-1a.privatecalico.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privatecalico.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privatecanal/kubernetes.tf b/tests/integration/update_cluster/privatecanal/kubernetes.tf index a23ec38fc8..3ae2a5a2ee 100644 --- a/tests/integration/update_cluster/privatecanal/kubernetes.tf +++ b/tests/integration/update_cluster/privatecanal/kubernetes.tf @@ -1195,6 +1195,8 @@ resource "aws_subnet" "us-test-1a-privatecanal-example-com" { "KubernetesCluster" = "privatecanal.example.com" "Name" = "us-test-1a.privatecanal.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privatecanal.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1208,6 +1210,7 @@ resource "aws_subnet" "utility-us-test-1a-privatecanal-example-com" { "KubernetesCluster" = "privatecanal.example.com" "Name" = "utility-us-test-1a.privatecanal.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privatecanal.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privatecilium/cloudformation.json b/tests/integration/update_cluster/privatecilium/cloudformation.json index 26c296dcad..382a9df3a5 100644 --- a/tests/integration/update_cluster/privatecilium/cloudformation.json +++ b/tests/integration/update_cluster/privatecilium/cloudformation.json @@ -1286,6 +1286,14 @@ "Key": "SubnetType", "Value": "Private" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/privatecilium.example.com", "Value": "owned" @@ -1318,6 +1326,10 @@ "Key": "SubnetType", "Value": "Utility" }, + { + "Key": "kops.k8s.io/instance-group/bastion", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/privatecilium.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/privatecilium/kubernetes.tf b/tests/integration/update_cluster/privatecilium/kubernetes.tf index 8ee04f5539..d13ca20827 100644 --- a/tests/integration/update_cluster/privatecilium/kubernetes.tf +++ b/tests/integration/update_cluster/privatecilium/kubernetes.tf @@ -1187,6 +1187,8 @@ resource "aws_subnet" "us-test-1a-privatecilium-example-com" { "KubernetesCluster" = "privatecilium.example.com" "Name" = "us-test-1a.privatecilium.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privatecilium.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1200,6 +1202,7 @@ resource "aws_subnet" "utility-us-test-1a-privatecilium-example-com" { "KubernetesCluster" = "privatecilium.example.com" "Name" = "utility-us-test-1a.privatecilium.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privatecilium.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privatecilium2/cloudformation.json b/tests/integration/update_cluster/privatecilium2/cloudformation.json index 5c0e984136..031dfa53c1 100644 --- a/tests/integration/update_cluster/privatecilium2/cloudformation.json +++ b/tests/integration/update_cluster/privatecilium2/cloudformation.json @@ -1234,6 +1234,14 @@ "Key": "SubnetType", "Value": "Private" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/privatecilium.example.com", "Value": "owned" @@ -1266,6 +1274,10 @@ "Key": "SubnetType", "Value": "Utility" }, + { + "Key": "kops.k8s.io/instance-group/bastion", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/privatecilium.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/privatecilium2/kubernetes.tf b/tests/integration/update_cluster/privatecilium2/kubernetes.tf index 809083c6a4..824ebd8d93 100644 --- a/tests/integration/update_cluster/privatecilium2/kubernetes.tf +++ b/tests/integration/update_cluster/privatecilium2/kubernetes.tf @@ -1189,6 +1189,8 @@ resource "aws_subnet" "us-test-1a-privatecilium-example-com" { "KubernetesCluster" = "privatecilium.example.com" "Name" = "us-test-1a.privatecilium.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privatecilium.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1204,6 +1206,7 @@ resource "aws_subnet" "utility-us-test-1a-privatecilium-example-com" { "KubernetesCluster" = "privatecilium.example.com" "Name" = "utility-us-test-1a.privatecilium.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privatecilium.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json b/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json index 06e7db2ee1..5327b2827e 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json +++ b/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json @@ -1286,6 +1286,14 @@ "Key": "SubnetType", "Value": "Private" }, + { + "Key": "kops.k8s.io/instance-group/master-us-test-1a", + "Value": "true" + }, + { + "Key": "kops.k8s.io/instance-group/nodes", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/privateciliumadvanced.example.com", "Value": "owned" @@ -1318,6 +1326,10 @@ "Key": "SubnetType", "Value": "Utility" }, + { + "Key": "kops.k8s.io/instance-group/bastion", + "Value": "true" + }, { "Key": "kubernetes.io/cluster/privateciliumadvanced.example.com", "Value": "owned" diff --git a/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf b/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf index 49d2455ca4..835ca045c0 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf +++ b/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf @@ -1219,6 +1219,8 @@ resource "aws_subnet" "us-test-1a-privateciliumadvanced-example-com" { "KubernetesCluster" = "privateciliumadvanced.example.com" "Name" = "us-test-1a.privateciliumadvanced.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privateciliumadvanced.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1232,6 +1234,7 @@ resource "aws_subnet" "utility-us-test-1a-privateciliumadvanced-example-com" { "KubernetesCluster" = "privateciliumadvanced.example.com" "Name" = "utility-us-test-1a.privateciliumadvanced.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privateciliumadvanced.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privatedns1/kubernetes.tf b/tests/integration/update_cluster/privatedns1/kubernetes.tf index 110a1ac88b..022fcdf503 100644 --- a/tests/integration/update_cluster/privatedns1/kubernetes.tf +++ b/tests/integration/update_cluster/privatedns1/kubernetes.tf @@ -1284,6 +1284,8 @@ resource "aws_subnet" "us-test-1a-privatedns1-example-com" { "Owner" = "John Doe" "SubnetType" = "Private" "foo/bar" = "fib+baz" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privatedns1.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1299,6 +1301,7 @@ resource "aws_subnet" "utility-us-test-1a-privatedns1-example-com" { "Owner" = "John Doe" "SubnetType" = "Utility" "foo/bar" = "fib+baz" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privatedns1.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privatedns2/kubernetes.tf b/tests/integration/update_cluster/privatedns2/kubernetes.tf index dbde6de8c0..0aa6e0011d 100644 --- a/tests/integration/update_cluster/privatedns2/kubernetes.tf +++ b/tests/integration/update_cluster/privatedns2/kubernetes.tf @@ -1165,6 +1165,8 @@ resource "aws_subnet" "us-test-1a-privatedns2-example-com" { "KubernetesCluster" = "privatedns2.example.com" "Name" = "us-test-1a.privatedns2.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privatedns2.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1178,6 +1180,7 @@ resource "aws_subnet" "utility-us-test-1a-privatedns2-example-com" { "KubernetesCluster" = "privatedns2.example.com" "Name" = "utility-us-test-1a.privatedns2.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privatedns2.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privateflannel/kubernetes.tf b/tests/integration/update_cluster/privateflannel/kubernetes.tf index 9b1ad4b671..54b1fe52a2 100644 --- a/tests/integration/update_cluster/privateflannel/kubernetes.tf +++ b/tests/integration/update_cluster/privateflannel/kubernetes.tf @@ -1187,6 +1187,8 @@ resource "aws_subnet" "us-test-1a-privateflannel-example-com" { "KubernetesCluster" = "privateflannel.example.com" "Name" = "us-test-1a.privateflannel.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privateflannel.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1200,6 +1202,7 @@ resource "aws_subnet" "utility-us-test-1a-privateflannel-example-com" { "KubernetesCluster" = "privateflannel.example.com" "Name" = "utility-us-test-1a.privateflannel.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privateflannel.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privatekopeio/kubernetes.tf b/tests/integration/update_cluster/privatekopeio/kubernetes.tf index 1237e75206..8d021169ad 100644 --- a/tests/integration/update_cluster/privatekopeio/kubernetes.tf +++ b/tests/integration/update_cluster/privatekopeio/kubernetes.tf @@ -1209,6 +1209,8 @@ resource "aws_subnet" "us-test-1a-privatekopeio-example-com" { "KubernetesCluster" = "privatekopeio.example.com" "Name" = "us-test-1a.privatekopeio.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privatekopeio.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1222,6 +1224,7 @@ resource "aws_subnet" "us-test-1b-privatekopeio-example-com" { "KubernetesCluster" = "privatekopeio.example.com" "Name" = "us-test-1b.privatekopeio.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privatekopeio.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1235,6 +1238,7 @@ resource "aws_subnet" "utility-us-test-1a-privatekopeio-example-com" { "KubernetesCluster" = "privatekopeio.example.com" "Name" = "utility-us-test-1a.privatekopeio.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privatekopeio.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/privateweave/kubernetes.tf b/tests/integration/update_cluster/privateweave/kubernetes.tf index c825f4a8f8..9bec614cf2 100644 --- a/tests/integration/update_cluster/privateweave/kubernetes.tf +++ b/tests/integration/update_cluster/privateweave/kubernetes.tf @@ -1187,6 +1187,8 @@ resource "aws_subnet" "us-test-1a-privateweave-example-com" { "KubernetesCluster" = "privateweave.example.com" "Name" = "us-test-1a.privateweave.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/privateweave.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1200,6 +1202,7 @@ resource "aws_subnet" "utility-us-test-1a-privateweave-example-com" { "KubernetesCluster" = "privateweave.example.com" "Name" = "utility-us-test-1a.privateweave.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/privateweave.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/public-jwks-apiserver/kubernetes.tf b/tests/integration/update_cluster/public-jwks-apiserver/kubernetes.tf index e61d7deb2d..93ba0369ec 100644 --- a/tests/integration/update_cluster/public-jwks-apiserver/kubernetes.tf +++ b/tests/integration/update_cluster/public-jwks-apiserver/kubernetes.tf @@ -888,12 +888,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { enable_resource_name_dns_a_record_on_launch = true private_dns_hostname_type_on_launch = "resource-name" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/tests/integration/update_cluster/shared_vpc/kubernetes.tf b/tests/integration/update_cluster/shared_vpc/kubernetes.tf index 70c64bd3b5..9bc5133080 100644 --- a/tests/integration/update_cluster/shared_vpc/kubernetes.tf +++ b/tests/integration/update_cluster/shared_vpc/kubernetes.tf @@ -755,12 +755,14 @@ resource "aws_subnet" "us-test-1a-sharedvpc-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "sharedvpc.example.com" - "Name" = "us-test-1a.sharedvpc.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/sharedvpc.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "sharedvpc.example.com" + "Name" = "us-test-1a.sharedvpc.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/sharedvpc.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = "vpc-12345678" } diff --git a/tests/integration/update_cluster/unmanaged/kubernetes.tf b/tests/integration/update_cluster/unmanaged/kubernetes.tf index b45c16bc47..1ddf897340 100644 --- a/tests/integration/update_cluster/unmanaged/kubernetes.tf +++ b/tests/integration/update_cluster/unmanaged/kubernetes.tf @@ -1095,11 +1095,13 @@ resource "aws_subnet" "us-test-1a-unmanaged-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "unmanaged.example.com" - "Name" = "us-test-1a.unmanaged.example.com" - "SubnetType" = "Private" - "kubernetes.io/cluster/unmanaged.example.com" = "owned" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "unmanaged.example.com" + "Name" = "us-test-1a.unmanaged.example.com" + "SubnetType" = "Private" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = "vpc-12345678" } @@ -1111,6 +1113,7 @@ resource "aws_subnet" "us-test-1b-unmanaged-example-com" { "KubernetesCluster" = "unmanaged.example.com" "Name" = "us-test-1b.unmanaged.example.com" "SubnetType" = "Private" + "kops.k8s.io/instance-group/nodes" = "true" "kubernetes.io/cluster/unmanaged.example.com" = "owned" "kubernetes.io/role/internal-elb" = "1" } @@ -1124,6 +1127,7 @@ resource "aws_subnet" "utility-us-test-1a-unmanaged-example-com" { "KubernetesCluster" = "unmanaged.example.com" "Name" = "utility-us-test-1a.unmanaged.example.com" "SubnetType" = "Utility" + "kops.k8s.io/instance-group/bastion" = "true" "kubernetes.io/cluster/unmanaged.example.com" = "owned" "kubernetes.io/role/elb" = "1" } diff --git a/tests/integration/update_cluster/vfs-said/kubernetes.tf b/tests/integration/update_cluster/vfs-said/kubernetes.tf index f68d30a6b7..8feffe431f 100644 --- a/tests/integration/update_cluster/vfs-said/kubernetes.tf +++ b/tests/integration/update_cluster/vfs-said/kubernetes.tf @@ -808,12 +808,14 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "us-test-1a.minimal.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - "kubernetes.io/role/internal-elb" = "1" + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" } vpc_id = aws_vpc.minimal-example-com.id } diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 7cef8e0f41..463f4223b6 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -1129,7 +1129,7 @@ spec: provider: launchTemplate: {{ $name }}.{{ ClusterName }} subnetSelector: - kubernetes.io/role/internal-elb: "1" + kops.k8s.io/instance-group/{{ $name }}: "*" kubernetes.io/cluster/{{ ClusterName }}: "*" ttlSecondsAfterEmpty: 300 {{ end }}