From c66180bc5834e918efe943f116dc14b12c97d7e9 Mon Sep 17 00:00:00 2001 From: Martin Tomes Date: Wed, 29 Apr 2020 14:53:17 +0200 Subject: [PATCH] Added support for instance interruption behavior --- k8s/crds/kops.k8s.io_instancegroups.yaml | 4 +++ pkg/apis/kops/instancegroup.go | 3 +++ pkg/apis/kops/v1alpha2/instancegroup.go | 3 +++ .../kops/v1alpha2/zz_generated.conversion.go | 2 ++ .../kops/v1alpha2/zz_generated.deepcopy.go | 5 ++++ pkg/apis/kops/validation/aws.go | 12 +++++++++ pkg/apis/kops/validation/aws_test.go | 26 +++++++++++++++++++ pkg/apis/kops/zz_generated.deepcopy.go | 5 ++++ pkg/model/awsmodel/autoscalinggroup.go | 3 +++ .../launch_templates/cloudformation.json | 1 + .../launch_templates/in-v1alpha2.yaml | 1 + .../launch_templates/kubernetes.tf | 5 ++-- .../pkg/fi/cloudup/awstasks/launchtemplate.go | 3 +++ .../launchtemplate_target_cloudformation.go | 7 +++-- ...unchtemplate_target_cloudformation_test.go | 18 +++++++------ .../launchtemplate_target_terraform.go | 7 +++-- .../launchtemplate_target_terraform_test.go | 22 +++++++++------- 17 files changed, 103 insertions(+), 24 deletions(-) diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index 96dc77271f..eff4fdbdcd 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -217,6 +217,10 @@ spec: image: description: Image is the instance (ami etc) we should use type: string + instanceInterruptionBehavior: + description: InstanceInterrutionBehavior defines if a spot instance + should be terminated, hibernated, or stopped after interruption + type: string instanceProtection: description: InstanceProtection makes new instances in an autoscaling group protected from scale in diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 37afbc4891..a952199f30 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -161,6 +161,9 @@ type InstanceGroupSpec struct { SysctlParameters []string `json:"sysctlParameters,omitempty"` // RollingUpdate defines the rolling-update behavior RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` + // InstanceInterrutionBehavior defines if a spot instance should be terminated, hibernated, + // or stopped after interruption + InstanceInterruptionBehavior *string `json:"instanceInterruptionBehavior,omitempty"` } const ( diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index 6812d58194..5489d9a2c4 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -157,6 +157,9 @@ type InstanceGroupSpec struct { SysctlParameters []string `json:"sysctlParameters,omitempty"` // RollingUpdate defines the rolling-update behavior RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` + // InstanceInterrutionBehavior defines if a spot instance should be terminated, hibernated, + // or stopped after interruption + InstanceInterruptionBehavior *string `json:"instanceInterruptionBehavior,omitempty"` } const ( diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 8b20fa1783..41466502c6 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -3366,6 +3366,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan } else { out.RollingUpdate = nil } + out.InstanceInterruptionBehavior = in.InstanceInterruptionBehavior return nil } @@ -3504,6 +3505,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I } else { out.RollingUpdate = nil } + out.InstanceInterruptionBehavior = in.InstanceInterruptionBehavior return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 7343504c69..1cf06326f6 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1787,6 +1787,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = new(RollingUpdate) (*in).DeepCopyInto(*out) } + if in.InstanceInterruptionBehavior != nil { + in, out := &in.InstanceInterruptionBehavior, &out.InstanceInterruptionBehavior + *out = new(string) + **out = **in + } return } diff --git a/pkg/apis/kops/validation/aws.go b/pkg/apis/kops/validation/aws.go index 00267b8931..e9a64867e8 100644 --- a/pkg/apis/kops/validation/aws.go +++ b/pkg/apis/kops/validation/aws.go @@ -51,6 +51,8 @@ func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList { allErrs = append(allErrs, awsValidateSpotDurationInMinute(field.NewPath(ig.GetName(), "spec", "spotDurationInMinutes"), ig)...) + allErrs = append(allErrs, awsValidateInstanceInterruptionBehavior(field.NewPath(ig.GetName(), "spec", "instanceInterruptionBehavior"), ig)...) + return allErrs } @@ -120,3 +122,13 @@ func awsValidateSpotDurationInMinute(fieldPath *field.Path, ig *kops.InstanceGro } return allErrs } + +func awsValidateInstanceInterruptionBehavior(fieldPath *field.Path, ig *kops.InstanceGroup) field.ErrorList { + allErrs := field.ErrorList{} + if ig.Spec.InstanceInterruptionBehavior != nil { + validInterruptionBehaviors := []string{"terminate", "hibernate", "stop"} + instanceInterruptionBehavior := *ig.Spec.InstanceInterruptionBehavior + allErrs = append(allErrs, IsValidValue(fieldPath, &instanceInterruptionBehavior, validInterruptionBehaviors)...) + } + return allErrs +} diff --git a/pkg/apis/kops/validation/aws_test.go b/pkg/apis/kops/validation/aws_test.go index 20cd30636c..9b496015c1 100644 --- a/pkg/apis/kops/validation/aws_test.go +++ b/pkg/apis/kops/validation/aws_test.go @@ -134,6 +134,32 @@ func TestValidateInstanceGroupSpec(t *testing.T) { }, ExpectedErrors: []string{}, }, + { + Input: kops.InstanceGroupSpec{ + InstanceInterruptionBehavior: fi.String("invalidValue"), + }, + ExpectedErrors: []string{ + "Unsupported value::test-nodes.spec.instanceInterruptionBehavior", + }, + }, + { + Input: kops.InstanceGroupSpec{ + InstanceInterruptionBehavior: fi.String("terminate"), + }, + ExpectedErrors: []string{}, + }, + { + Input: kops.InstanceGroupSpec{ + InstanceInterruptionBehavior: fi.String("hibernate"), + }, + ExpectedErrors: []string{}, + }, + { + Input: kops.InstanceGroupSpec{ + InstanceInterruptionBehavior: fi.String("stop"), + }, + ExpectedErrors: []string{}, + }, } for _, g := range grid { ig := &kops.InstanceGroup{ diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 629784685e..8c7b4f4551 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1953,6 +1953,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = new(RollingUpdate) (*in).DeepCopyInto(*out) } + if in.InstanceInterruptionBehavior != nil { + in, out := &in.InstanceInterruptionBehavior, &out.InstanceInterruptionBehavior + *out = new(string) + **out = **in + } return } diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index dcc611b4a6..ab2438ce8c 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -134,6 +134,9 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde if ig.Spec.SpotDurationInMinutes != nil { lt.SpotDurationInMinutes = ig.Spec.SpotDurationInMinutes } + if ig.Spec.InstanceInterruptionBehavior != nil { + lt.InstanceInterruptionBehavior = ig.Spec.InstanceInterruptionBehavior + } return lt, nil } diff --git a/tests/integration/update_cluster/launch_templates/cloudformation.json b/tests/integration/update_cluster/launch_templates/cloudformation.json index cfb4a0e841..2033fc77f1 100644 --- a/tests/integration/update_cluster/launch_templates/cloudformation.json +++ b/tests/integration/update_cluster/launch_templates/cloudformation.json @@ -606,6 +606,7 @@ "MarketType": "spot", "SpotOptions": { "BlockDurationMinutes": 120, + "InstanceInterruptionBehavior": "hibernate", "MaxPrice": "0.1" } }, diff --git a/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml b/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml index 44f663c28f..8d58f039a9 100644 --- a/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml @@ -73,6 +73,7 @@ spec: instanceProtection: true maxPrice: "0.1" spotDurationInMinutes: 120 + instanceInterruptionBehavior: "hibernate" subnets: - us-test-1b --- diff --git a/tests/integration/update_cluster/launch_templates/kubernetes.tf b/tests/integration/update_cluster/launch_templates/kubernetes.tf index f7abc21cc6..e7cefff781 100644 --- a/tests/integration/update_cluster/launch_templates/kubernetes.tf +++ b/tests/integration/update_cluster/launch_templates/kubernetes.tf @@ -530,8 +530,9 @@ resource "aws_launch_template" "nodes-launchtemplates-example-com" { instance_market_options { market_type = "spot" spot_options { - block_duration_minutes = 120 - max_price = "0.1" + block_duration_minutes = 120 + instance_interruption_behavior = "hibernate" + max_price = "0.1" } } instance_type = "t3.medium" diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate.go index a55907e0d5..7c1a7a37e2 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate.go @@ -70,6 +70,9 @@ type LaunchTemplate struct { Tenancy *string // UserData is the user data configuration UserData *fi.ResourceHolder + // InstanceInterrutionBehavior defines if a spot instance should be terminated, hibernated, + // or stopped after interruption + InstanceInterruptionBehavior *string } var ( diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go index 39f3a0d75f..c621f75c2b 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go @@ -64,8 +64,8 @@ type cloudformationLaunchTemplateIAMProfile struct { type cloudformationLaunchTemplateMarketOptionsSpotOptions struct { // BlockDurationMinutes is required duration in minutes. This value must be a multiple of 60. BlockDurationMinutes *int64 `json:"BlockDurationMinutes,omitempty"` - // InstancesInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate - InstancesInterruptionBehavior *string `json:"InstancesInterruptionBehavior,omitempty"` + // InstanceInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate + InstanceInterruptionBehavior *string `json:"InstanceInterruptionBehavior,omitempty"` // MaxPrice is the maximum hourly price you're willing to pay for the Spot Instances MaxPrice *string `json:"MaxPrice,omitempty"` // SpotInstanceType is the Spot Instance request type. Can be one-time, or persistent @@ -185,6 +185,9 @@ func (t *LaunchTemplate) RenderCloudformation(target *cloudformation.Cloudformat if e.SpotDurationInMinutes != nil { marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes } + if e.InstanceInterruptionBehavior != nil { + marketSpotOptions.InstanceInterruptionBehavior = e.InstanceInterruptionBehavior + } launchTemplateData.MarketOptions = &cloudformationLaunchTemplateMarketOptions{MarketType: fi.String("spot"), SpotOptions: &marketSpotOptions} } diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go index 427f96a583..aa14b6587e 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go @@ -31,14 +31,15 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) { IAMInstanceProfile: &IAMInstanceProfile{ Name: fi.String("nodes"), }, - ID: fi.String("test-11"), - InstanceMonitoring: fi.Bool(true), - InstanceType: fi.String("t2.medium"), - RootVolumeOptimization: fi.Bool(true), - RootVolumeIops: fi.Int64(100), - RootVolumeSize: fi.Int64(64), - SpotPrice: "10", - SpotDurationInMinutes: fi.Int64(120), + ID: fi.String("test-11"), + InstanceMonitoring: fi.Bool(true), + InstanceType: fi.String("t2.medium"), + RootVolumeOptimization: fi.Bool(true), + RootVolumeIops: fi.Int64(100), + RootVolumeSize: fi.Int64(64), + SpotPrice: "10", + SpotDurationInMinutes: fi.Int64(120), + InstanceInterruptionBehavior: fi.String("hibernate"), SSHKey: &SSHKey{ Name: fi.String("mykey"), }, @@ -67,6 +68,7 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) { "MarketType": "spot", "SpotOptions": { "BlockDurationMinutes": 120, + "InstanceInterruptionBehavior": "hibernate", "MaxPrice": "10" } }, diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go index ebe60d4d7a..276a8d27c7 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go @@ -61,8 +61,8 @@ type terraformLaunchTemplateIAMProfile struct { type terraformLaunchTemplateMarketOptionsSpotOptions struct { // BlockDurationMinutes is required duration in minutes. This value must be a multiple of 60. BlockDurationMinutes *int64 `json:"block_duration_minutes,omitempty" cty:"block_duration_minutes"` - // InstancesInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate - InstancesInterruptionBehavior *string `json:"instances_interruption_behavior,omitempty" cty:"instances_interruption_behavior"` + // InstanceInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate + InstanceInterruptionBehavior *string `json:"instance_interruption_behavior,omitempty" cty:"instance_interruption_behavior"` // MaxPrice is the maximum hourly price you're willing to pay for the Spot Instances MaxPrice *string `json:"max_price,omitempty" cty:"max_price"` // SpotInstanceType is the Spot Instance request type. Can be one-time, or persistent @@ -183,6 +183,9 @@ func (t *LaunchTemplate) RenderTerraform(target *terraform.TerraformTarget, a, e if e.SpotDurationInMinutes != nil { marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes } + if e.InstanceInterruptionBehavior != nil { + marketSpotOptions.InstanceInterruptionBehavior = e.InstanceInterruptionBehavior + } tf.MarketOptions = []*terraformLaunchTemplateMarketOptions{ { MarketType: fi.String("spot"), diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go index e0016a4e66..3007127922 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go @@ -31,14 +31,15 @@ func TestLaunchTemplateTerraformRender(t *testing.T) { IAMInstanceProfile: &IAMInstanceProfile{ Name: fi.String("nodes"), }, - ID: fi.String("test-11"), - InstanceMonitoring: fi.Bool(true), - InstanceType: fi.String("t2.medium"), - SpotPrice: "0.1", - SpotDurationInMinutes: fi.Int64(60), - RootVolumeOptimization: fi.Bool(true), - RootVolumeIops: fi.Int64(100), - RootVolumeSize: fi.Int64(64), + ID: fi.String("test-11"), + InstanceMonitoring: fi.Bool(true), + InstanceType: fi.String("t2.medium"), + SpotPrice: "0.1", + SpotDurationInMinutes: fi.Int64(60), + InstanceInterruptionBehavior: fi.String("hibernate"), + RootVolumeOptimization: fi.Bool(true), + RootVolumeIops: fi.Int64(100), + RootVolumeSize: fi.Int64(64), SSHKey: &SSHKey{ Name: fi.String("newkey"), PublicKey: fi.WrapResource(fi.NewStringResource("newkey")), @@ -61,8 +62,9 @@ resource "aws_launch_template" "test" { instance_market_options { market_type = "spot" spot_options { - block_duration_minutes = 60 - max_price = "0.1" + block_duration_minutes = 60 + instance_interruption_behavior = "hibernate" + max_price = "0.1" } } instance_type = "t2.medium"