diff --git a/cloudmock/aws/mockautoscaling/group.go b/cloudmock/aws/mockautoscaling/group.go index 19e1ee098c..9be02f7bd7 100644 --- a/cloudmock/aws/mockautoscaling/group.go +++ b/cloudmock/aws/mockautoscaling/group.go @@ -74,6 +74,7 @@ func (m *MockAutoscaling) CreateAutoScalingGroup(input *autoscaling.CreateAutoSc TargetGroupARNs: input.TargetGroupARNs, TerminationPolicies: input.TerminationPolicies, VPCZoneIdentifier: input.VPCZoneIdentifier, + MaxInstanceLifetime: input.MaxInstanceLifetime, } if input.LaunchTemplate != nil { diff --git a/docs/instance_groups.md b/docs/instance_groups.md index 15762426fe..d4c3b0f697 100644 --- a/docs/instance_groups.md +++ b/docs/instance_groups.md @@ -320,3 +320,19 @@ spec: httpPutResponseHopLimit: 1 httpTokens: required ``` + +## maxInstanceLifetime (AWS Only) + +{{ kops_feature_table(kops_added_default='1.24') }} + +The maximum instance lifetime specifies the maximum amount of time (in go duration [format](https://pkg.go.dev/time#ParseDuration)) that an instance can be in service before it is terminated and replaced. +A common use case might be a requirement to replace your instances on a schedule because of internal security policies or external compliance controls. +In other words, this feature helps you to put a bit of ephemerality in your cluster. +You must specify a value of at least 24h (86,400 seconds). To clear a previously set value, specify a new value of 0. + +The following configuration enables a maximum instance lifetime to two days. + +```yaml +spec: + maxInstanceLifetime: "48h" +``` diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index 8bb427a37b..6739a1341a 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -754,6 +754,11 @@ spec: manager: description: Manager determines what is managing the node lifecycle type: string + maxInstanceLifetime: + description: MaxInstanceLifetime to the maximum amount of time, in + seconds, that an instance can be in service. Value expected must + be in form of duration ("ms", "s", "m", "h") + type: string maxPrice: description: MaxPrice indicates this is a spot-pricing group, with the specified value as our max-price bid diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index e43b10e506..3869b86132 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -196,6 +196,9 @@ type InstanceGroupSpec struct { Packages []string `json:"packages,omitempty"` // GuestAccelerators configures additional accelerators GuestAccelerators []AcceleratorConfig `json:"guestAccelerators,omitempty"` + // MaxInstanceLifetime to the maximum amount of time, in seconds, that an instance can be in service. + // Value expected must be in form of duration ("ms", "s", "m", "h") + MaxInstanceLifetime *metav1.Duration `json:"maxInstanceLifetime,omitempty"` } const ( diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index 972d51016e..fae9b43a1c 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -162,6 +162,9 @@ type InstanceGroupSpec struct { Packages []string `json:"packages,omitempty"` // GuestAccelerators configures additional accelerators GuestAccelerators []AcceleratorConfig `json:"guestAccelerators,omitempty"` + // MaxInstanceLifetime to the maximum amount of time, in seconds, that an instance can be in service. + // Value expected must be in form of duration ("ms", "s", "m", "h") + MaxInstanceLifetime *metav1.Duration `json:"maxInstanceLifetime,omitempty"` } // InstanceMetadataOptions defines the EC2 instance metadata service options (AWS Only) diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 47a56f9a36..6cacd82bc6 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -4471,6 +4471,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan } else { out.GuestAccelerators = nil } + out.MaxInstanceLifetime = in.MaxInstanceLifetime return nil } @@ -4654,6 +4655,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I } else { out.GuestAccelerators = nil } + out.MaxInstanceLifetime = in.MaxInstanceLifetime return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 91a04183fc..d548bc8fda 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2534,6 +2534,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = make([]AcceleratorConfig, len(*in)) copy(*out, *in) } + if in.MaxInstanceLifetime != nil { + in, out := &in.MaxInstanceLifetime, &out.MaxInstanceLifetime + *out = new(v1.Duration) + **out = **in + } return } diff --git a/pkg/apis/kops/v1alpha3/instancegroup.go b/pkg/apis/kops/v1alpha3/instancegroup.go index ed15dcbb3b..d2e65b5e54 100644 --- a/pkg/apis/kops/v1alpha3/instancegroup.go +++ b/pkg/apis/kops/v1alpha3/instancegroup.go @@ -159,6 +159,9 @@ type InstanceGroupSpec struct { Packages []string `json:"packages,omitempty"` // GuestAccelerators configures additional accelerators GuestAccelerators []AcceleratorConfig `json:"guestAccelerators,omitempty"` + // MaxInstanceLifetime to the maximum amount of time, in seconds, that an instance can be in service. + // Value expected must be in form of duration ("ms", "s", "m", "h") + MaxInstanceLifetime *metav1.Duration `json:"maxInstanceLifetime,omitempty"` } // InstanceMetadataOptions defines the EC2 instance metadata service options (AWS Only) diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index 5c297c5ff9..175ef12cbc 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -4610,6 +4610,7 @@ func autoConvert_v1alpha3_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan } else { out.GuestAccelerators = nil } + out.MaxInstanceLifetime = in.MaxInstanceLifetime return nil } @@ -4793,6 +4794,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha3_InstanceGroupSpec(in *kops.I } else { out.GuestAccelerators = nil } + out.MaxInstanceLifetime = in.MaxInstanceLifetime return nil } diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index 213635c546..ea778aae3d 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -2540,6 +2540,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = make([]AcceleratorConfig, len(*in)) copy(*out, *in) } + if in.MaxInstanceLifetime != nil { + in, out := &in.MaxInstanceLifetime, &out.MaxInstanceLifetime + *out = new(v1.Duration) + **out = **in + } return } diff --git a/pkg/apis/kops/validation/aws.go b/pkg/apis/kops/validation/aws.go index a9410267b1..3940305b95 100644 --- a/pkg/apis/kops/validation/aws.go +++ b/pkg/apis/kops/validation/aws.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/service/ec2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kops/pkg/apis/kops" @@ -100,6 +101,21 @@ func awsValidateInstanceGroup(ig *kops.InstanceGroup, cloud awsup.AWSCloud) fiel allErrs = append(allErrs, awsValidateCPUCredits(field.NewPath("spec"), &ig.Spec, cloud)...) } + if ig.Spec.MaxInstanceLifetime != nil { + allErrs = append(allErrs, awsValidateMaximumInstanceLifetime(field.NewPath(ig.GetName(), "spec"), ig.Spec.MaxInstanceLifetime)...) + } + + return allErrs +} + +func awsValidateMaximumInstanceLifetime(fieldPath *field.Path, maxInstanceLifetime *metav1.Duration) field.ErrorList { + allErrs := field.ErrorList{} + const minMaxInstanceLifetime = 86400 + lifetimeSec := int64(maxInstanceLifetime.Seconds()) + if lifetimeSec != 0 && lifetimeSec < minMaxInstanceLifetime { + allErrs = append(allErrs, field.Invalid(fieldPath.Child("maxInstanceLifetime"), maxInstanceLifetime, fmt.Sprintf("max instance lifetime must be greater than %d or equal to 0", int64(minMaxInstanceLifetime)))) + } + return allErrs } diff --git a/pkg/apis/kops/validation/aws_test.go b/pkg/apis/kops/validation/aws_test.go index 5b359035a6..4da94dfbea 100644 --- a/pkg/apis/kops/validation/aws_test.go +++ b/pkg/apis/kops/validation/aws_test.go @@ -18,6 +18,7 @@ package validation import ( "testing" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -26,6 +27,7 @@ import ( "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kops/pkg/apis/kops" ) @@ -204,6 +206,26 @@ func TestValidateInstanceGroupSpec(t *testing.T) { }, ExpectedErrors: []string{}, }, + { + Input: kops.InstanceGroupSpec{ + MaxInstanceLifetime: &metav1.Duration{Duration: time.Duration(100 * float64(time.Second))}, + }, + ExpectedErrors: []string{ + "Invalid value::test-nodes.spec.maxInstanceLifetime", + }, + }, + { + Input: kops.InstanceGroupSpec{ + MaxInstanceLifetime: &metav1.Duration{Duration: time.Duration(86400 * float64(time.Second))}, + }, + ExpectedErrors: []string{}, + }, + { + Input: kops.InstanceGroupSpec{ + MaxInstanceLifetime: &metav1.Duration{Duration: time.Duration(0 * float64(time.Second))}, + }, + ExpectedErrors: []string{}, + }, } cloud := awsup.BuildMockAWSCloud("us-east-1", "abc") mockEC2 := &mockec2.MockEC2{} diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 0ee6c18c15..4c6846964b 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2703,6 +2703,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = make([]AcceleratorConfig, len(*in)) copy(*out, *in) } + if in.MaxInstanceLifetime != nil { + in, out := &in.MaxInstanceLifetime, &out.MaxInstanceLifetime + *out = new(v1.Duration) + **out = **in + } return } diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index d9e6ee82ed..06748bb431 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -547,5 +547,11 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil } } + if ig.Spec.MaxInstanceLifetime != nil { + lifetimeSec := int64(ig.Spec.MaxInstanceLifetime.Seconds()) + t.MaxInstanceLifetime = fi.Int64(lifetimeSec) + } else { + t.MaxInstanceLifetime = fi.Int64(0) + } return t, nil } diff --git a/tests/integration/update_cluster/apiservernodes/kubernetes.tf b/tests/integration/update_cluster/apiservernodes/kubernetes.tf index 700649894c..e0d0c2e20c 100644 --- a/tests/integration/update_cluster/apiservernodes/kubernetes.tf +++ b/tests/integration/update_cluster/apiservernodes/kubernetes.tf @@ -111,6 +111,7 @@ resource "aws_autoscaling_group" "apiserver-apiservers-minimal-example-com" { id = aws_launch_template.apiserver-apiservers-minimal-example-com.id version = aws_launch_template.apiserver-apiservers-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 @@ -160,6 +161,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -229,6 +231,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/aws-lb-controller/kubernetes.tf b/tests/integration/update_cluster/aws-lb-controller/kubernetes.tf index b0ec818a8f..c5d6909d69 100644 --- a/tests/integration/update_cluster/aws-lb-controller/kubernetes.tf +++ b/tests/integration/update_cluster/aws-lb-controller/kubernetes.tf @@ -121,6 +121,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -185,6 +186,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf index 06dc51dc05..2483c933d3 100644 --- a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf +++ b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf @@ -122,6 +122,7 @@ resource "aws_autoscaling_group" "bastion-bastionuserdata-example-com" { version = aws_launch_template.bastion-bastionuserdata-example-com.latest_version } load_balancers = [aws_elb.bastion-bastionuserdata-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -172,6 +173,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-bastionuserdata-exam version = aws_launch_template.master-us-test-1a-masters-bastionuserdata-example-com.latest_version } load_balancers = [aws_elb.api-bastionuserdata-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -236,6 +238,7 @@ resource "aws_autoscaling_group" "nodes-bastionuserdata-example-com" { id = aws_launch_template.nodes-bastionuserdata-example-com.id version = aws_launch_template.nodes-bastionuserdata-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/complex/kubernetes.tf b/tests/integration/update_cluster/complex/kubernetes.tf index 271ab0e5d7..96aa9c013c 100644 --- a/tests/integration/update_cluster/complex/kubernetes.tf +++ b/tests/integration/update_cluster/complex/kubernetes.tf @@ -109,6 +109,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-complex-example-com" version = aws_launch_template.master-us-test-1a-masters-complex-example-com.latest_version } load_balancers = ["my-external-lb-1"] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -185,6 +186,7 @@ resource "aws_autoscaling_group" "nodes-complex-example-com" { version = aws_launch_template.nodes-complex-example-com.latest_version } load_balancers = ["my-external-lb-1"] + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/compress/kubernetes.tf b/tests/integration/update_cluster/compress/kubernetes.tf index 591265823b..8053f1f60c 100644 --- a/tests/integration/update_cluster/compress/kubernetes.tf +++ b/tests/integration/update_cluster/compress/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-compress-example-com id = aws_launch_template.master-us-test-1a-masters-compress-example-com.id version = aws_launch_template.master-us-test-1a-masters-compress-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-compress-example-com" { id = aws_launch_template.nodes-compress-example-com.id version = aws_launch_template.nodes-compress-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/digit/kubernetes.tf b/tests/integration/update_cluster/digit/kubernetes.tf index b26d9b633f..24178834a4 100644 --- a/tests/integration/update_cluster/digit/kubernetes.tf +++ b/tests/integration/update_cluster/digit/kubernetes.tf @@ -121,6 +121,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-123-example-com" { id = aws_launch_template.master-us-test-1a-masters-123-example-com.id version = aws_launch_template.master-us-test-1a-masters-123-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -185,6 +186,7 @@ resource "aws_autoscaling_group" "nodes-123-example-com" { id = aws_launch_template.nodes-123-example-com.id version = aws_launch_template.nodes-123-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/existing_iam/kubernetes.tf b/tests/integration/update_cluster/existing_iam/kubernetes.tf index 99f3aaa403..4c0fb2135f 100644 --- a/tests/integration/update_cluster/existing_iam/kubernetes.tf +++ b/tests/integration/update_cluster/existing_iam/kubernetes.tf @@ -81,6 +81,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-existing-iam-example id = aws_launch_template.master-us-test-1a-masters-existing-iam-example-com.id version = aws_launch_template.master-us-test-1a-masters-existing-iam-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -145,6 +146,7 @@ resource "aws_autoscaling_group" "master-us-test-1b-masters-existing-iam-example id = aws_launch_template.master-us-test-1b-masters-existing-iam-example-com.id version = aws_launch_template.master-us-test-1b-masters-existing-iam-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -209,6 +211,7 @@ resource "aws_autoscaling_group" "master-us-test-1c-masters-existing-iam-example id = aws_launch_template.master-us-test-1c-masters-existing-iam-example-com.id version = aws_launch_template.master-us-test-1c-masters-existing-iam-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -273,6 +276,7 @@ resource "aws_autoscaling_group" "nodes-existing-iam-example-com" { id = aws_launch_template.nodes-existing-iam-example-com.id version = aws_launch_template.nodes-existing-iam-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/existing_sg/kubernetes.tf b/tests/integration/update_cluster/existing_sg/kubernetes.tf index a7125e2e14..6fcd5b3910 100644 --- a/tests/integration/update_cluster/existing_sg/kubernetes.tf +++ b/tests/integration/update_cluster/existing_sg/kubernetes.tf @@ -102,6 +102,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-existingsg-example-c version = aws_launch_template.master-us-test-1a-masters-existingsg-example-com.latest_version } load_balancers = [aws_elb.api-existingsg-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -167,6 +168,7 @@ resource "aws_autoscaling_group" "master-us-test-1b-masters-existingsg-example-c version = aws_launch_template.master-us-test-1b-masters-existingsg-example-com.latest_version } load_balancers = [aws_elb.api-existingsg-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -232,6 +234,7 @@ resource "aws_autoscaling_group" "master-us-test-1c-masters-existingsg-example-c version = aws_launch_template.master-us-test-1c-masters-existingsg-example-com.latest_version } load_balancers = [aws_elb.api-existingsg-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -296,6 +299,7 @@ resource "aws_autoscaling_group" "nodes-existingsg-example-com" { id = aws_launch_template.nodes-existingsg-example-com.id version = aws_launch_template.nodes-existingsg-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/external_dns/kubernetes.tf b/tests/integration/update_cluster/external_dns/kubernetes.tf index 9a51a9f629..a33607b580 100644 --- a/tests/integration/update_cluster/external_dns/kubernetes.tf +++ b/tests/integration/update_cluster/external_dns/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf b/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf index cea5028f20..6438e161e6 100644 --- a/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf +++ b/tests/integration/update_cluster/external_dns_irsa/kubernetes.tf @@ -111,6 +111,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -175,6 +176,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/externallb/kubernetes.tf b/tests/integration/update_cluster/externallb/kubernetes.tf index 163ddf9810..ed56d099d2 100644 --- a/tests/integration/update_cluster/externallb/kubernetes.tf +++ b/tests/integration/update_cluster/externallb/kubernetes.tf @@ -92,6 +92,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-externallb-example-c version = aws_launch_template.master-us-test-1a-masters-externallb-example-com.latest_version } load_balancers = ["my-external-elb-1", "my-external-elb-2", "my-external-elb-3"] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -158,6 +159,7 @@ resource "aws_autoscaling_group" "nodes-externallb-example-com" { version = aws_launch_template.nodes-externallb-example-com.latest_version } load_balancers = ["my-external-elb-1"] + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/externalpolicies/kubernetes.tf b/tests/integration/update_cluster/externalpolicies/kubernetes.tf index 1249d3d114..bc79501794 100644 --- a/tests/integration/update_cluster/externalpolicies/kubernetes.tf +++ b/tests/integration/update_cluster/externalpolicies/kubernetes.tf @@ -92,6 +92,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-externalpolicies-exa version = aws_launch_template.master-us-test-1a-masters-externalpolicies-example-com.latest_version } load_balancers = [aws_elb.api-externalpolicies-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -166,6 +167,7 @@ resource "aws_autoscaling_group" "nodes-externalpolicies-example-com" { id = aws_launch_template.nodes-externalpolicies-example-com.id version = aws_launch_template.nodes-externalpolicies-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/ha/kubernetes.tf b/tests/integration/update_cluster/ha/kubernetes.tf index 55eeeb5f37..8409703cd5 100644 --- a/tests/integration/update_cluster/ha/kubernetes.tf +++ b/tests/integration/update_cluster/ha/kubernetes.tf @@ -101,6 +101,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-ha-example-com" { id = aws_launch_template.master-us-test-1a-masters-ha-example-com.id version = aws_launch_template.master-us-test-1a-masters-ha-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -165,6 +166,7 @@ resource "aws_autoscaling_group" "master-us-test-1b-masters-ha-example-com" { id = aws_launch_template.master-us-test-1b-masters-ha-example-com.id version = aws_launch_template.master-us-test-1b-masters-ha-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -229,6 +231,7 @@ resource "aws_autoscaling_group" "master-us-test-1c-masters-ha-example-com" { id = aws_launch_template.master-us-test-1c-masters-ha-example-com.id version = aws_launch_template.master-us-test-1c-masters-ha-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -293,6 +296,7 @@ resource "aws_autoscaling_group" "nodes-ha-example-com" { id = aws_launch_template.nodes-ha-example-com.id version = aws_launch_template.nodes-ha-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/irsa/kubernetes.tf b/tests/integration/update_cluster/irsa/kubernetes.tf index a6b06919d5..d7d04be341 100644 --- a/tests/integration/update_cluster/irsa/kubernetes.tf +++ b/tests/integration/update_cluster/irsa/kubernetes.tf @@ -131,6 +131,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -195,6 +196,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index 7c67530929..e2a810553e 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -121,6 +121,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -185,6 +186,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa/kubernetes.tf b/tests/integration/update_cluster/many-addons-ccm-irsa/kubernetes.tf index f30e933351..309c107e0c 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-ccm-irsa/kubernetes.tf @@ -161,6 +161,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -225,6 +226,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa23/kubernetes.tf b/tests/integration/update_cluster/many-addons-ccm-irsa23/kubernetes.tf index 867ba4479e..b2474ff657 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa23/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-ccm-irsa23/kubernetes.tf @@ -161,6 +161,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -225,6 +226,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa24/kubernetes.tf b/tests/integration/update_cluster/many-addons-ccm-irsa24/kubernetes.tf index fe8c4fe44e..2d554df728 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa24/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/kubernetes.tf @@ -161,6 +161,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -215,6 +216,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/many-addons-ccm/kubernetes.tf b/tests/integration/update_cluster/many-addons-ccm/kubernetes.tf index c573108915..920b979e8d 100644 --- a/tests/integration/update_cluster/many-addons-ccm/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-ccm/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/many-addons/kubernetes.tf b/tests/integration/update_cluster/many-addons/kubernetes.tf index 4400d1c64e..cab123a5db 100644 --- a/tests/integration/update_cluster/many-addons/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/minimal-1.23/kubernetes.tf b/tests/integration/update_cluster/minimal-1.23/kubernetes.tf index 5fdbc169eb..991a7dfd20 100644 --- a/tests/integration/update_cluster/minimal-1.23/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-1.23/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -160,6 +161,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 diff --git a/tests/integration/update_cluster/minimal-1.24/kubernetes.tf b/tests/integration/update_cluster/minimal-1.24/kubernetes.tf index 4c358a9e19..dd0d4a2a52 100644 --- a/tests/integration/update_cluster/minimal-1.24/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-1.24/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -150,6 +151,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 diff --git a/tests/integration/update_cluster/minimal-gp3/kubernetes.tf b/tests/integration/update_cluster/minimal-gp3/kubernetes.tf index 24a2477e32..e298f4f952 100644 --- a/tests/integration/update_cluster/minimal-gp3/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-gp3/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/minimal-ipv6-calico/kubernetes.tf b/tests/integration/update_cluster/minimal-ipv6-calico/kubernetes.tf index 912f2c3cf4..9bdee79dd7 100644 --- a/tests/integration/update_cluster/minimal-ipv6-calico/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-ipv6-calico/kubernetes.tf @@ -96,6 +96,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-ipv6-example id = aws_launch_template.master-us-test-1a-masters-minimal-ipv6-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-ipv6-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -156,6 +157,7 @@ resource "aws_autoscaling_group" "nodes-minimal-ipv6-example-com" { id = aws_launch_template.nodes-minimal-ipv6-example-com.id version = aws_launch_template.nodes-minimal-ipv6-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 diff --git a/tests/integration/update_cluster/minimal-ipv6-cilium/kubernetes.tf b/tests/integration/update_cluster/minimal-ipv6-cilium/kubernetes.tf index 3d9affb909..0288221f7b 100644 --- a/tests/integration/update_cluster/minimal-ipv6-cilium/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-ipv6-cilium/kubernetes.tf @@ -96,6 +96,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-ipv6-example id = aws_launch_template.master-us-test-1a-masters-minimal-ipv6-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-ipv6-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -161,6 +162,7 @@ resource "aws_autoscaling_group" "nodes-minimal-ipv6-example-com" { id = aws_launch_template.nodes-minimal-ipv6-example-com.id version = aws_launch_template.nodes-minimal-ipv6-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf b/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf index 6f1dd76c56..75490125c9 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf @@ -126,6 +126,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-ipv6-example id = aws_launch_template.master-us-test-1a-masters-minimal-ipv6-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-ipv6-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -191,6 +192,7 @@ resource "aws_autoscaling_group" "nodes-minimal-ipv6-example-com" { id = aws_launch_template.nodes-minimal-ipv6-example-com.id version = aws_launch_template.nodes-minimal-ipv6-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/minimal-ipv6/kubernetes.tf b/tests/integration/update_cluster/minimal-ipv6/kubernetes.tf index 4ab9c14868..2524c29565 100644 --- a/tests/integration/update_cluster/minimal-ipv6/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-ipv6/kubernetes.tf @@ -96,6 +96,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-ipv6-example id = aws_launch_template.master-us-test-1a-masters-minimal-ipv6-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-ipv6-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -161,6 +162,7 @@ resource "aws_autoscaling_group" "nodes-minimal-ipv6-example-com" { id = aws_launch_template.nodes-minimal-ipv6-example-com.id version = aws_launch_template.nodes-minimal-ipv6-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/minimal-longclustername/kubernetes.tf b/tests/integration/update_cluster/minimal-longclustername/kubernetes.tf index 09ed2814ad..2bdc3251e4 100644 --- a/tests/integration/update_cluster/minimal-longclustername/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-longclustername/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-this-is-truly-a-real id = aws_launch_template.master-us-test-1a-masters-this-is-truly-a-really-really-long-cluster-name-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-this-is-truly-a-really-really-long-cluster-name-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-this-is-truly-a-really-really-long-clust id = aws_launch_template.nodes-this-is-truly-a-really-really-long-cluster-name-minimal-example-com.id version = aws_launch_template.nodes-this-is-truly-a-really-really-long-cluster-name-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/minimal-warmpool/kubernetes.tf b/tests/integration/update_cluster/minimal-warmpool/kubernetes.tf index 56b447c339..11f4a2152b 100644 --- a/tests/integration/update_cluster/minimal-warmpool/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-warmpool/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-warmpool-exa id = aws_launch_template.master-us-test-1a-masters-minimal-warmpool-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-warmpool-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-minimal-warmpool-example-com" { id = aws_launch_template.nodes-minimal-warmpool-example-com.id version = aws_launch_template.nodes-minimal-warmpool-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/minimal/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal/in-v1alpha2.yaml index 039b51880c..74969ec313 100644 --- a/tests/integration/update_cluster/minimal/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/minimal/in-v1alpha2.yaml @@ -51,6 +51,7 @@ spec: machineType: t2.medium maxSize: 2 minSize: 2 + maxInstanceLifetime: 48h20m role: Node subnets: - us-test-1a @@ -70,6 +71,7 @@ spec: machineType: m3.medium maxSize: 1 minSize: 1 + maxInstanceLifetime: "0" role: Master subnets: - us-test-1a diff --git a/tests/integration/update_cluster/minimal/kubernetes.tf b/tests/integration/update_cluster/minimal/kubernetes.tf index 2538e7d984..e2d72862cd 100644 --- a/tests/integration/update_cluster/minimal/kubernetes.tf +++ b/tests/integration/update_cluster/minimal/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 174000 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/minimal_gossip/kubernetes.tf b/tests/integration/update_cluster/minimal_gossip/kubernetes.tf index dd87d89a0b..0200f0d0f8 100644 --- a/tests/integration/update_cluster/minimal_gossip/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gossip/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-k8s-local" { id = aws_launch_template.master-us-test-1a-masters-minimal-k8s-local.id version = aws_launch_template.master-us-test-1a-masters-minimal-k8s-local.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-minimal-k8s-local" { id = aws_launch_template.nodes-minimal-k8s-local.id version = aws_launch_template.nodes-minimal-k8s-local.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/minimal_gossip_irsa/kubernetes.tf b/tests/integration/update_cluster/minimal_gossip_irsa/kubernetes.tf index c4141141b4..b27fb9bc93 100644 --- a/tests/integration/update_cluster/minimal_gossip_irsa/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gossip_irsa/kubernetes.tf @@ -101,6 +101,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-k8s-local" { id = aws_launch_template.master-us-test-1a-masters-minimal-k8s-local.id version = aws_launch_template.master-us-test-1a-masters-minimal-k8s-local.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -165,6 +166,7 @@ resource "aws_autoscaling_group" "nodes-minimal-k8s-local" { id = aws_launch_template.nodes-minimal-k8s-local.id version = aws_launch_template.nodes-minimal-k8s-local.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/mixed_instances/kubernetes.tf b/tests/integration/update_cluster/mixed_instances/kubernetes.tf index d27ed7dbfc..add3ae15f4 100644 --- a/tests/integration/update_cluster/mixed_instances/kubernetes.tf +++ b/tests/integration/update_cluster/mixed_instances/kubernetes.tf @@ -101,6 +101,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-mixedinstances-examp id = aws_launch_template.master-us-test-1a-masters-mixedinstances-example-com.id version = aws_launch_template.master-us-test-1a-masters-mixedinstances-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -165,6 +166,7 @@ resource "aws_autoscaling_group" "master-us-test-1b-masters-mixedinstances-examp id = aws_launch_template.master-us-test-1b-masters-mixedinstances-example-com.id version = aws_launch_template.master-us-test-1b-masters-mixedinstances-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -229,6 +231,7 @@ resource "aws_autoscaling_group" "master-us-test-1c-masters-mixedinstances-examp id = aws_launch_template.master-us-test-1c-masters-mixedinstances-example-com.id version = aws_launch_template.master-us-test-1c-masters-mixedinstances-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -288,10 +291,11 @@ resource "aws_autoscaling_group" "master-us-test-1c-masters-mixedinstances-examp } resource "aws_autoscaling_group" "nodes-mixedinstances-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + max_instance_lifetime = 0 + max_size = 2 + metrics_granularity = "1Minute" + min_size = 2 mixed_instances_policy { instances_distribution { on_demand_percentage_above_base_capacity = 5 diff --git a/tests/integration/update_cluster/mixed_instances_spot/kubernetes.tf b/tests/integration/update_cluster/mixed_instances_spot/kubernetes.tf index 6d35b6c3c6..b6aed877cd 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/kubernetes.tf +++ b/tests/integration/update_cluster/mixed_instances_spot/kubernetes.tf @@ -101,6 +101,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-mixedinstances-examp id = aws_launch_template.master-us-test-1a-masters-mixedinstances-example-com.id version = aws_launch_template.master-us-test-1a-masters-mixedinstances-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -165,6 +166,7 @@ resource "aws_autoscaling_group" "master-us-test-1b-masters-mixedinstances-examp id = aws_launch_template.master-us-test-1b-masters-mixedinstances-example-com.id version = aws_launch_template.master-us-test-1b-masters-mixedinstances-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -229,6 +231,7 @@ resource "aws_autoscaling_group" "master-us-test-1c-masters-mixedinstances-examp id = aws_launch_template.master-us-test-1c-masters-mixedinstances-example-com.id version = aws_launch_template.master-us-test-1c-masters-mixedinstances-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -288,10 +291,11 @@ resource "aws_autoscaling_group" "master-us-test-1c-masters-mixedinstances-examp } resource "aws_autoscaling_group" "nodes-mixedinstances-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + max_instance_lifetime = 0 + max_size = 2 + metrics_granularity = "1Minute" + min_size = 2 mixed_instances_policy { instances_distribution { on_demand_percentage_above_base_capacity = 5 diff --git a/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf b/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf index a3fd9ff8c2..d6969b08c6 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf +++ b/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-nthsqsresources-long id = aws_launch_template.master-us-test-1a-masters-nthsqsresources-longclustername-example-com.id version = aws_launch_template.master-us-test-1a-masters-nthsqsresources-longclustername-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -160,6 +161,7 @@ resource "aws_autoscaling_group" "nodes-nthsqsresources-longclustername-example- id = aws_launch_template.nodes-nthsqsresources-longclustername-example-com.id version = aws_launch_template.nodes-nthsqsresources-longclustername-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/nvidia/kubernetes.tf b/tests/integration/update_cluster/nvidia/kubernetes.tf index b0139ae874..08e4889eb3 100644 --- a/tests/integration/update_cluster/nvidia/kubernetes.tf +++ b/tests/integration/update_cluster/nvidia/kubernetes.tf @@ -91,6 +91,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -155,6 +156,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/private-shared-ip/kubernetes.tf b/tests/integration/update_cluster/private-shared-ip/kubernetes.tf index 8e4d6c6c92..c250c1ec0c 100644 --- a/tests/integration/update_cluster/private-shared-ip/kubernetes.tf +++ b/tests/integration/update_cluster/private-shared-ip/kubernetes.tf @@ -117,6 +117,7 @@ resource "aws_autoscaling_group" "bastion-private-shared-ip-example-com" { version = aws_launch_template.bastion-private-shared-ip-example-com.latest_version } load_balancers = [aws_elb.bastion-private-shared-ip-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -167,6 +168,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-private-shared-ip-ex version = aws_launch_template.master-us-test-1a-masters-private-shared-ip-example-com.latest_version } load_balancers = [aws_elb.api-private-shared-ip-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -231,6 +233,7 @@ resource "aws_autoscaling_group" "nodes-private-shared-ip-example-com" { id = aws_launch_template.nodes-private-shared-ip-example-com.id version = aws_launch_template.nodes-private-shared-ip-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/private-shared-subnet/kubernetes.tf b/tests/integration/update_cluster/private-shared-subnet/kubernetes.tf index 2ef81b3d90..a4e8778781 100644 --- a/tests/integration/update_cluster/private-shared-subnet/kubernetes.tf +++ b/tests/integration/update_cluster/private-shared-subnet/kubernetes.tf @@ -112,6 +112,7 @@ resource "aws_autoscaling_group" "bastion-private-shared-subnet-example-com" { version = aws_launch_template.bastion-private-shared-subnet-example-com.latest_version } load_balancers = [aws_elb.bastion-private-shared-subnet-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -162,6 +163,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-private-shared-subne version = aws_launch_template.master-us-test-1a-masters-private-shared-subnet-example-com.latest_version } load_balancers = [aws_elb.api-private-shared-subnet-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -226,6 +228,7 @@ resource "aws_autoscaling_group" "nodes-private-shared-subnet-example-com" { id = aws_launch_template.nodes-private-shared-subnet-example-com.id version = aws_launch_template.nodes-private-shared-subnet-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privatecalico/kubernetes.tf b/tests/integration/update_cluster/privatecalico/kubernetes.tf index d76e3ac9b4..8d55c104f1 100644 --- a/tests/integration/update_cluster/privatecalico/kubernetes.tf +++ b/tests/integration/update_cluster/privatecalico/kubernetes.tf @@ -122,6 +122,7 @@ resource "aws_autoscaling_group" "bastion-privatecalico-example-com" { version = aws_launch_template.bastion-privatecalico-example-com.latest_version } load_balancers = [aws_elb.bastion-privatecalico-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -167,6 +168,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatecalico-exampl version = aws_launch_template.master-us-test-1a-masters-privatecalico-example-com.latest_version } load_balancers = [aws_elb.api-privatecalico-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -221,6 +223,7 @@ resource "aws_autoscaling_group" "nodes-privatecalico-example-com" { id = aws_launch_template.nodes-privatecalico-example-com.id version = aws_launch_template.nodes-privatecalico-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privatecanal/kubernetes.tf b/tests/integration/update_cluster/privatecanal/kubernetes.tf index 92c382021b..a23ec38fc8 100644 --- a/tests/integration/update_cluster/privatecanal/kubernetes.tf +++ b/tests/integration/update_cluster/privatecanal/kubernetes.tf @@ -122,6 +122,7 @@ resource "aws_autoscaling_group" "bastion-privatecanal-example-com" { version = aws_launch_template.bastion-privatecanal-example-com.latest_version } load_balancers = [aws_elb.bastion-privatecanal-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -172,6 +173,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatecanal-example version = aws_launch_template.master-us-test-1a-masters-privatecanal-example-com.latest_version } load_balancers = [aws_elb.api-privatecanal-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -236,6 +238,7 @@ resource "aws_autoscaling_group" "nodes-privatecanal-example-com" { id = aws_launch_template.nodes-privatecanal-example-com.id version = aws_launch_template.nodes-privatecanal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privatecilium/kubernetes.tf b/tests/integration/update_cluster/privatecilium/kubernetes.tf index a1376a7b91..8ee04f5539 100644 --- a/tests/integration/update_cluster/privatecilium/kubernetes.tf +++ b/tests/integration/update_cluster/privatecilium/kubernetes.tf @@ -122,6 +122,7 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" { version = aws_launch_template.bastion-privatecilium-example-com.latest_version } load_balancers = [aws_elb.bastion-privatecilium-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -172,6 +173,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatecilium-exampl version = aws_launch_template.master-us-test-1a-masters-privatecilium-example-com.latest_version } load_balancers = [aws_elb.api-privatecilium-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -236,6 +238,7 @@ resource "aws_autoscaling_group" "nodes-privatecilium-example-com" { id = aws_launch_template.nodes-privatecilium-example-com.id version = aws_launch_template.nodes-privatecilium-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privatecilium2/kubernetes.tf b/tests/integration/update_cluster/privatecilium2/kubernetes.tf index d92ad9ed1c..809083c6a4 100644 --- a/tests/integration/update_cluster/privatecilium2/kubernetes.tf +++ b/tests/integration/update_cluster/privatecilium2/kubernetes.tf @@ -122,6 +122,7 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" { version = aws_launch_template.bastion-privatecilium-example-com.latest_version } load_balancers = [aws_elb.bastion-privatecilium-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -167,6 +168,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatecilium-exampl version = aws_launch_template.master-us-test-1a-masters-privatecilium-example-com.latest_version } load_balancers = [aws_elb.api-privatecilium-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -221,6 +223,7 @@ resource "aws_autoscaling_group" "nodes-privatecilium-example-com" { id = aws_launch_template.nodes-privatecilium-example-com.id version = aws_launch_template.nodes-privatecilium-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf b/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf index 89ff9daecf..49d2455ca4 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf +++ b/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf @@ -122,6 +122,7 @@ resource "aws_autoscaling_group" "bastion-privateciliumadvanced-example-com" { version = aws_launch_template.bastion-privateciliumadvanced-example-com.latest_version } load_balancers = [aws_elb.bastion-privateciliumadvanced-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -172,6 +173,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privateciliumadvance version = aws_launch_template.master-us-test-1a-masters-privateciliumadvanced-example-com.latest_version } load_balancers = [aws_elb.api-privateciliumadvanced-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -236,6 +238,7 @@ resource "aws_autoscaling_group" "nodes-privateciliumadvanced-example-com" { id = aws_launch_template.nodes-privateciliumadvanced-example-com.id version = aws_launch_template.nodes-privateciliumadvanced-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privatedns1/kubernetes.tf b/tests/integration/update_cluster/privatedns1/kubernetes.tf index 0dadef3834..110a1ac88b 100644 --- a/tests/integration/update_cluster/privatedns1/kubernetes.tf +++ b/tests/integration/update_cluster/privatedns1/kubernetes.tf @@ -122,6 +122,7 @@ resource "aws_autoscaling_group" "bastion-privatedns1-example-com" { version = aws_launch_template.bastion-privatedns1-example-com.latest_version } load_balancers = [aws_elb.bastion-privatedns1-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -182,6 +183,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatedns1-example- version = aws_launch_template.master-us-test-1a-masters-privatedns1-example-com.latest_version } load_balancers = [aws_elb.api-privatedns1-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -256,6 +258,7 @@ resource "aws_autoscaling_group" "nodes-privatedns1-example-com" { id = aws_launch_template.nodes-privatedns1-example-com.id version = aws_launch_template.nodes-privatedns1-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privatedns2/kubernetes.tf b/tests/integration/update_cluster/privatedns2/kubernetes.tf index 4e6f26c2d3..dbde6de8c0 100644 --- a/tests/integration/update_cluster/privatedns2/kubernetes.tf +++ b/tests/integration/update_cluster/privatedns2/kubernetes.tf @@ -117,6 +117,7 @@ resource "aws_autoscaling_group" "bastion-privatedns2-example-com" { version = aws_launch_template.bastion-privatedns2-example-com.latest_version } load_balancers = [aws_elb.bastion-privatedns2-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -167,6 +168,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatedns2-example- version = aws_launch_template.master-us-test-1a-masters-privatedns2-example-com.latest_version } load_balancers = [aws_elb.api-privatedns2-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -231,6 +233,7 @@ resource "aws_autoscaling_group" "nodes-privatedns2-example-com" { id = aws_launch_template.nodes-privatedns2-example-com.id version = aws_launch_template.nodes-privatedns2-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privateflannel/kubernetes.tf b/tests/integration/update_cluster/privateflannel/kubernetes.tf index 22671aa0ea..9b1ad4b671 100644 --- a/tests/integration/update_cluster/privateflannel/kubernetes.tf +++ b/tests/integration/update_cluster/privateflannel/kubernetes.tf @@ -122,6 +122,7 @@ resource "aws_autoscaling_group" "bastion-privateflannel-example-com" { version = aws_launch_template.bastion-privateflannel-example-com.latest_version } load_balancers = [aws_elb.bastion-privateflannel-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -172,6 +173,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privateflannel-examp version = aws_launch_template.master-us-test-1a-masters-privateflannel-example-com.latest_version } load_balancers = [aws_elb.api-privateflannel-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -236,6 +238,7 @@ resource "aws_autoscaling_group" "nodes-privateflannel-example-com" { id = aws_launch_template.nodes-privateflannel-example-com.id version = aws_launch_template.nodes-privateflannel-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privatekopeio/kubernetes.tf b/tests/integration/update_cluster/privatekopeio/kubernetes.tf index 590466dbe1..1237e75206 100644 --- a/tests/integration/update_cluster/privatekopeio/kubernetes.tf +++ b/tests/integration/update_cluster/privatekopeio/kubernetes.tf @@ -137,6 +137,7 @@ resource "aws_autoscaling_group" "bastion-privatekopeio-example-com" { version = aws_launch_template.bastion-privatekopeio-example-com.latest_version } load_balancers = [aws_elb.bastion-privatekopeio-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -187,6 +188,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatekopeio-exampl version = aws_launch_template.master-us-test-1a-masters-privatekopeio-example-com.latest_version } load_balancers = [aws_elb.api-privatekopeio-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -251,6 +253,7 @@ resource "aws_autoscaling_group" "nodes-privatekopeio-example-com" { id = aws_launch_template.nodes-privatekopeio-example-com.id version = aws_launch_template.nodes-privatekopeio-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/privateweave/kubernetes.tf b/tests/integration/update_cluster/privateweave/kubernetes.tf index 70125f72a2..c825f4a8f8 100644 --- a/tests/integration/update_cluster/privateweave/kubernetes.tf +++ b/tests/integration/update_cluster/privateweave/kubernetes.tf @@ -122,6 +122,7 @@ resource "aws_autoscaling_group" "bastion-privateweave-example-com" { version = aws_launch_template.bastion-privateweave-example-com.latest_version } load_balancers = [aws_elb.bastion-privateweave-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -172,6 +173,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privateweave-example version = aws_launch_template.master-us-test-1a-masters-privateweave-example-com.latest_version } load_balancers = [aws_elb.api-privateweave-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -236,6 +238,7 @@ resource "aws_autoscaling_group" "nodes-privateweave-example-com" { id = aws_launch_template.nodes-privateweave-example-com.id version = aws_launch_template.nodes-privateweave-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/public-jwks-apiserver/kubernetes.tf b/tests/integration/update_cluster/public-jwks-apiserver/kubernetes.tf index 791ebd330c..e61d7deb2d 100644 --- a/tests/integration/update_cluster/public-jwks-apiserver/kubernetes.tf +++ b/tests/integration/update_cluster/public-jwks-apiserver/kubernetes.tf @@ -131,6 +131,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -185,6 +186,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/shared_subnet/kubernetes.tf b/tests/integration/update_cluster/shared_subnet/kubernetes.tf index 79dc875c83..9ea322ca2c 100644 --- a/tests/integration/update_cluster/shared_subnet/kubernetes.tf +++ b/tests/integration/update_cluster/shared_subnet/kubernetes.tf @@ -86,6 +86,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-sharedsubnet-example id = aws_launch_template.master-us-test-1a-masters-sharedsubnet-example-com.id version = aws_launch_template.master-us-test-1a-masters-sharedsubnet-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -150,6 +151,7 @@ resource "aws_autoscaling_group" "nodes-sharedsubnet-example-com" { id = aws_launch_template.nodes-sharedsubnet-example-com.id version = aws_launch_template.nodes-sharedsubnet-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/shared_vpc/kubernetes.tf b/tests/integration/update_cluster/shared_vpc/kubernetes.tf index 5e0a0f688c..70c64bd3b5 100644 --- a/tests/integration/update_cluster/shared_vpc/kubernetes.tf +++ b/tests/integration/update_cluster/shared_vpc/kubernetes.tf @@ -86,6 +86,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-sharedvpc-example-co id = aws_launch_template.master-us-test-1a-masters-sharedvpc-example-com.id version = aws_launch_template.master-us-test-1a-masters-sharedvpc-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -150,6 +151,7 @@ resource "aws_autoscaling_group" "nodes-sharedvpc-example-com" { id = aws_launch_template.nodes-sharedvpc-example-com.id version = aws_launch_template.nodes-sharedvpc-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/unmanaged/kubernetes.tf b/tests/integration/update_cluster/unmanaged/kubernetes.tf index f2eb54b2fa..b45c16bc47 100644 --- a/tests/integration/update_cluster/unmanaged/kubernetes.tf +++ b/tests/integration/update_cluster/unmanaged/kubernetes.tf @@ -117,6 +117,7 @@ resource "aws_autoscaling_group" "bastion-unmanaged-example-com" { version = aws_launch_template.bastion-unmanaged-example-com.latest_version } load_balancers = [aws_elb.bastion-unmanaged-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -167,6 +168,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-unmanaged-example-co version = aws_launch_template.master-us-test-1a-masters-unmanaged-example-com.latest_version } load_balancers = [aws_elb.api-unmanaged-example-com.id] + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -231,6 +233,7 @@ resource "aws_autoscaling_group" "nodes-unmanaged-example-com" { id = aws_launch_template.nodes-unmanaged-example-com.id version = aws_launch_template.nodes-unmanaged-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/tests/integration/update_cluster/vfs-said/kubernetes.tf b/tests/integration/update_cluster/vfs-said/kubernetes.tf index b9fd3c7884..f68d30a6b7 100644 --- a/tests/integration/update_cluster/vfs-said/kubernetes.tf +++ b/tests/integration/update_cluster/vfs-said/kubernetes.tf @@ -101,6 +101,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" min_size = 1 @@ -165,6 +166,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { id = aws_launch_template.nodes-minimal-example-com.id version = aws_launch_template.nodes-minimal-example-com.latest_version } + max_instance_lifetime = 0 max_size = 2 metrics_granularity = "1Minute" min_size = 2 diff --git a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go index 086aee8773..84d383cd64 100644 --- a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go +++ b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go @@ -37,7 +37,7 @@ import ( // CloudTagInstanceGroupRolePrefix is a cloud tag that defines the instance role const CloudTagInstanceGroupRolePrefix = "k8s.io/role/" -// AutoscalingGroup provdes the definition for a autoscaling group in aws +// AutoscalingGroup provides the definition for a autoscaling group in aws // +kops:fitask type AutoscalingGroup struct { // Name is the name of the ASG @@ -53,6 +53,8 @@ type AutoscalingGroup struct { LaunchTemplate *LaunchTemplate // LoadBalancers is a list of elastic load balancer names to add to the autoscaling group LoadBalancers []*ClassicLoadBalancer + // MaxInstanceLifetime is the maximum amount of time, in seconds, that an instance can be in service. + MaxInstanceLifetime *int64 // MaxSize is the max number of nodes in asg MaxSize *int64 // Metrics is a collection of metrics to monitor @@ -110,9 +112,17 @@ func (e *AutoscalingGroup) Find(c *fi.Context) (*AutoscalingGroup, error) { } actual := &AutoscalingGroup{ - Name: g.AutoScalingGroupName, - MaxSize: g.MaxSize, - MinSize: g.MinSize, + Name: g.AutoScalingGroupName, + MaxSize: g.MaxSize, + MinSize: g.MinSize, + MaxInstanceLifetime: g.MaxInstanceLifetime, + } + + // Use 0 as default value when api returns nil (same as model) + if g.MaxInstanceLifetime == nil { + actual.MaxInstanceLifetime = fi.Int64(0) + } else { + actual.MaxInstanceLifetime = g.MaxInstanceLifetime } actual.LoadBalancers = []*ClassicLoadBalancer{} @@ -344,6 +354,13 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos VPCZoneIdentifier: fi.String(strings.Join(e.AutoscalingGroupSubnets(), ",")), } + //On ASG creation 0 value is forbidden + if fi.Int64Value(e.MaxInstanceLifetime) == 0 { + request.MaxInstanceLifetime = nil + } else { + request.MaxInstanceLifetime = e.MaxInstanceLifetime + } + for _, k := range e.LoadBalancers { if k.LoadBalancerName == nil { lbDesc, err := t.Cloud.FindELBByNameTag(fi.StringValue(k.GetName())) @@ -523,6 +540,13 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos changes.Subnets = nil } + if changes.MaxInstanceLifetime != nil { + request.MaxInstanceLifetime = e.MaxInstanceLifetime + changes.MaxInstanceLifetime = nil + } else { + request.MaxInstanceLifetime = fi.Int64(0) + } + var updateTagsRequest *autoscaling.CreateOrUpdateTagsInput var deleteTagsRequest *autoscaling.DeleteTagsInput if changes.Tags != nil { @@ -891,17 +915,19 @@ type terraformAutoscalingGroup struct { InstanceProtection *bool `cty:"protect_from_scale_in"` LoadBalancers []*terraformWriter.Literal `cty:"load_balancers"` TargetGroupARNs []*terraformWriter.Literal `cty:"target_group_arns"` + MaxInstanceLifetime *int64 `cty:"max_instance_lifetime"` } // RenderTerraform is responsible for rendering the terraform codebase func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *AutoscalingGroup) error { tf := &terraformAutoscalingGroup{ - Name: e.Name, - MinSize: e.MinSize, - MaxSize: e.MaxSize, - MetricsGranularity: e.Granularity, - EnabledMetrics: aws.StringSlice(e.Metrics), - InstanceProtection: e.InstanceProtection, + Name: e.Name, + MinSize: e.MinSize, + MaxSize: e.MaxSize, + MetricsGranularity: e.Granularity, + EnabledMetrics: aws.StringSlice(e.Metrics), + InstanceProtection: e.InstanceProtection, + MaxInstanceLifetime: e.MaxInstanceLifetime, } for _, s := range e.Subnets {