Merge pull request #9024 from tomesm/support_launch_template

Added Launch Template support for instance interruption behavior

k8s/crds/kops.k8s.io_instancegroups.yaml

@@ -217,6 +217,10 @@ spec:
               image:
                 description: Image is the instance (ami etc) we should use
                 type: string
+              instanceInterruptionBehavior:
+                description: InstanceInterruptionBehavior defines if a spot instance
+                  should be terminated, hibernated, or stopped after interruption
+                type: string
               instanceProtection:
                 description: InstanceProtection makes new instances in an autoscaling
                   group protected from scale in

pkg/apis/kops/instancegroup.go

@@ -161,6 +161,9 @@ type InstanceGroupSpec struct {
 	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 	// RollingUpdate defines the rolling-update behavior
 	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+	// InstanceInterruptionBehavior defines if a spot instance should be terminated, hibernated,
+	// or stopped after interruption
+	InstanceInterruptionBehavior *string `json:"instanceInterruptionBehavior,omitempty"`
 }
 
 const (

pkg/apis/kops/v1alpha2/instancegroup.go

@@ -157,6 +157,9 @@ type InstanceGroupSpec struct {
 	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 	// RollingUpdate defines the rolling-update behavior
 	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+	// InstanceInterruptionBehavior defines if a spot instance should be terminated, hibernated,
+	// or stopped after interruption
+	InstanceInterruptionBehavior *string `json:"instanceInterruptionBehavior,omitempty"`
 }
 
 const (

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -3370,6 +3370,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 	} else {
 		out.RollingUpdate = nil
 	}
+	out.InstanceInterruptionBehavior = in.InstanceInterruptionBehavior
 	return nil
 }
 
@@ -3508,6 +3509,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
 	} else {
 		out.RollingUpdate = nil
 	}
+	out.InstanceInterruptionBehavior = in.InstanceInterruptionBehavior
 	return nil
 }
 

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -1797,6 +1797,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(RollingUpdate)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.InstanceInterruptionBehavior != nil {
+		in, out := &in.InstanceInterruptionBehavior, &out.InstanceInterruptionBehavior
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/apis/kops/validation/aws.go

@@ -51,6 +51,8 @@ func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList {
 
 	allErrs = append(allErrs, awsValidateSpotDurationInMinute(field.NewPath(ig.GetName(), "spec", "spotDurationInMinutes"), ig)...)
 
+	allErrs = append(allErrs, awsValidateInstanceInterruptionBehavior(field.NewPath(ig.GetName(), "spec", "instanceInterruptionBehavior"), ig)...)
+
 	return allErrs
 }
 
@@ -120,3 +122,13 @@ func awsValidateSpotDurationInMinute(fieldPath *field.Path, ig *kops.InstanceGro
 	}
 	return allErrs
 }
+
+func awsValidateInstanceInterruptionBehavior(fieldPath *field.Path, ig *kops.InstanceGroup) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if ig.Spec.InstanceInterruptionBehavior != nil {
+		validInterruptionBehaviors := []string{"terminate", "hibernate", "stop"}
+		instanceInterruptionBehavior := *ig.Spec.InstanceInterruptionBehavior
+		allErrs = append(allErrs, IsValidValue(fieldPath, &instanceInterruptionBehavior, validInterruptionBehaviors)...)
+	}
+	return allErrs
+}

pkg/apis/kops/validation/aws_test.go

@@ -134,6 +134,32 @@ func TestValidateInstanceGroupSpec(t *testing.T) {
 			},
 			ExpectedErrors: []string{},
 		},
+		{
+			Input: kops.InstanceGroupSpec{
+				InstanceInterruptionBehavior: fi.String("invalidValue"),
+			},
+			ExpectedErrors: []string{
+				"Unsupported value::test-nodes.spec.instanceInterruptionBehavior",
+			},
+		},
+		{
+			Input: kops.InstanceGroupSpec{
+				InstanceInterruptionBehavior: fi.String("terminate"),
+			},
+			ExpectedErrors: []string{},
+		},
+		{
+			Input: kops.InstanceGroupSpec{
+				InstanceInterruptionBehavior: fi.String("hibernate"),
+			},
+			ExpectedErrors: []string{},
+		},
+		{
+			Input: kops.InstanceGroupSpec{
+				InstanceInterruptionBehavior: fi.String("stop"),
+			},
+			ExpectedErrors: []string{},
+		},
 	}
 	for _, g := range grid {
 		ig := &kops.InstanceGroup{

pkg/apis/kops/zz_generated.deepcopy.go

@@ -1963,6 +1963,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(RollingUpdate)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.InstanceInterruptionBehavior != nil {
+		in, out := &in.InstanceInterruptionBehavior, &out.InstanceInterruptionBehavior
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/model/awsmodel/autoscalinggroup.go

@@ -134,6 +134,9 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde
 	if ig.Spec.SpotDurationInMinutes != nil {
 		lt.SpotDurationInMinutes = ig.Spec.SpotDurationInMinutes
 	}
+	if ig.Spec.InstanceInterruptionBehavior != nil {
+		lt.InstanceInterruptionBehavior = ig.Spec.InstanceInterruptionBehavior
+	}
 	return lt, nil
 }
 

tests/integration/update_cluster/launch_templates/cloudformation.json

@@ -606,6 +606,7 @@
             "MarketType": "spot",
             "SpotOptions": {
               "BlockDurationMinutes": 120,
+              "InstanceInterruptionBehavior": "hibernate",
               "MaxPrice": "0.1"
             }
           },

tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml

@@ -73,6 +73,7 @@ spec:
   instanceProtection: true
   maxPrice: "0.1"
   spotDurationInMinutes: 120
+  instanceInterruptionBehavior: "hibernate"
   subnets:
   - us-test-1b
 ---

tests/integration/update_cluster/launch_templates/kubernetes.tf

@@ -530,8 +530,9 @@ resource "aws_launch_template" "nodes-launchtemplates-example-com" {
   instance_market_options {
     market_type = "spot"
     spot_options {
-      block_duration_minutes = 120
+      block_duration_minutes         = 120
-      max_price              = "0.1"
+      instance_interruption_behavior = "hibernate"
+      max_price                      = "0.1"
     }
   }
   instance_type = "t3.medium"

upup/pkg/fi/cloudup/awstasks/launchtemplate.go

@@ -70,6 +70,9 @@ type LaunchTemplate struct {
 	Tenancy *string
 	// UserData is the user data configuration
 	UserData *fi.ResourceHolder
+	// InstanceInterruptionBehavior defines if a spot instance should be terminated, hibernated,
+	// or stopped after interruption
+	InstanceInterruptionBehavior *string
 }
 
 var (

upup/pkg/fi/cloudup/awstasks/launchtemplate_target_api.go

@@ -128,7 +128,15 @@ func (t *LaunchTemplate) RenderAWS(c *awsup.AWSAPITarget, a, ep, changes *Launch
 		}
 		lc.UserData = aws.String(base64.StdEncoding.EncodeToString(d))
 	}
-
+	// @step: add instanceInterruptionBehavior
+	if t.InstanceInterruptionBehavior != nil {
+		s := &ec2.LaunchTemplateSpotMarketOptionsRequest{
+			InstanceInterruptionBehavior: t.InstanceInterruptionBehavior,
+		}
+		lc.InstanceMarketOptions = &ec2.LaunchTemplateInstanceMarketOptionsRequest{
+			SpotOptions: s,
+		}
+	}
 	// @step: attempt to create the launch template
 	err = func() error {
 		for attempt := 0; attempt < 10; attempt++ {
@@ -223,6 +231,10 @@ func (t *LaunchTemplate) Find(c *fi.Context) (*LaunchTemplate, error) {
 	if lt.LaunchTemplateData.IamInstanceProfile != nil {
 		actual.IAMInstanceProfile = &IAMInstanceProfile{Name: lt.LaunchTemplateData.IamInstanceProfile.Name}
 	}
+	// @step: add instanceInterruptionBehavior if there is one
+	if lt.LaunchTemplateData.InstanceMarketOptions != nil && lt.LaunchTemplateData.InstanceMarketOptions.SpotOptions != nil {
+		actual.InstanceInterruptionBehavior = lt.LaunchTemplateData.InstanceMarketOptions.SpotOptions.InstanceInterruptionBehavior
+	}
 
 	// @step: get the image is order to find out the root device name as using the index
 	// is not variable, under conditions they move

upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go

@@ -64,8 +64,8 @@ type cloudformationLaunchTemplateIAMProfile struct {
 type cloudformationLaunchTemplateMarketOptionsSpotOptions struct {
 	// BlockDurationMinutes is required duration in minutes. This value must be a multiple of 60.
 	BlockDurationMinutes *int64 `json:"BlockDurationMinutes,omitempty"`
-	// InstancesInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate
+	// InstanceInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate
-	InstancesInterruptionBehavior *string `json:"InstancesInterruptionBehavior,omitempty"`
+	InstanceInterruptionBehavior *string `json:"InstanceInterruptionBehavior,omitempty"`
 	// MaxPrice is the maximum hourly price you're willing to pay for the Spot Instances
 	MaxPrice *string `json:"MaxPrice,omitempty"`
 	// SpotInstanceType is the Spot Instance request type. Can be one-time, or persistent
@@ -185,6 +185,9 @@ func (t *LaunchTemplate) RenderCloudformation(target *cloudformation.Cloudformat
 		if e.SpotDurationInMinutes != nil {
 			marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes
 		}
+		if e.InstanceInterruptionBehavior != nil {
+			marketSpotOptions.InstanceInterruptionBehavior = e.InstanceInterruptionBehavior
+		}
 		launchTemplateData.MarketOptions = &cloudformationLaunchTemplateMarketOptions{MarketType: fi.String("spot"), SpotOptions: &marketSpotOptions}
 	}
 

upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go

@@ -31,14 +31,15 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) {
 				IAMInstanceProfile: &IAMInstanceProfile{
 					Name: fi.String("nodes"),
 				},
-				ID:                     fi.String("test-11"),
+				ID:                           fi.String("test-11"),
-				InstanceMonitoring:     fi.Bool(true),
+				InstanceMonitoring:           fi.Bool(true),
-				InstanceType:           fi.String("t2.medium"),
+				InstanceType:                 fi.String("t2.medium"),
-				RootVolumeOptimization: fi.Bool(true),
+				RootVolumeOptimization:       fi.Bool(true),
-				RootVolumeIops:         fi.Int64(100),
+				RootVolumeIops:               fi.Int64(100),
-				RootVolumeSize:         fi.Int64(64),
+				RootVolumeSize:               fi.Int64(64),
-				SpotPrice:              "10",
+				SpotPrice:                    "10",
-				SpotDurationInMinutes:  fi.Int64(120),
+				SpotDurationInMinutes:        fi.Int64(120),
+				InstanceInterruptionBehavior: fi.String("hibernate"),
 				SSHKey: &SSHKey{
 					Name: fi.String("mykey"),
 				},
@@ -67,6 +68,7 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) {
             "MarketType": "spot",
             "SpotOptions": {
               "BlockDurationMinutes": 120,
+              "InstanceInterruptionBehavior": "hibernate",
               "MaxPrice": "10"
             }
           },

upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go

@@ -61,8 +61,8 @@ type terraformLaunchTemplateIAMProfile struct {
 type terraformLaunchTemplateMarketOptionsSpotOptions struct {
 	// BlockDurationMinutes is required duration in minutes. This value must be a multiple of 60.
 	BlockDurationMinutes *int64 `json:"block_duration_minutes,omitempty" cty:"block_duration_minutes"`
-	// InstancesInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate
+	// InstanceInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate
-	InstancesInterruptionBehavior *string `json:"instances_interruption_behavior,omitempty" cty:"instances_interruption_behavior"`
+	InstanceInterruptionBehavior *string `json:"instance_interruption_behavior,omitempty" cty:"instance_interruption_behavior"`
 	// MaxPrice is the maximum hourly price you're willing to pay for the Spot Instances
 	MaxPrice *string `json:"max_price,omitempty" cty:"max_price"`
 	// SpotInstanceType is the Spot Instance request type. Can be one-time, or persistent
@@ -183,6 +183,9 @@ func (t *LaunchTemplate) RenderTerraform(target *terraform.TerraformTarget, a, e
 		if e.SpotDurationInMinutes != nil {
 			marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes
 		}
+		if e.InstanceInterruptionBehavior != nil {
+			marketSpotOptions.InstanceInterruptionBehavior = e.InstanceInterruptionBehavior
+		}
 		tf.MarketOptions = []*terraformLaunchTemplateMarketOptions{
 			{
 				MarketType:  fi.String("spot"),

upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go

@@ -31,14 +31,15 @@ func TestLaunchTemplateTerraformRender(t *testing.T) {
 				IAMInstanceProfile: &IAMInstanceProfile{
 					Name: fi.String("nodes"),
 				},
-				ID:                     fi.String("test-11"),
+				ID:                           fi.String("test-11"),
-				InstanceMonitoring:     fi.Bool(true),
+				InstanceMonitoring:           fi.Bool(true),
-				InstanceType:           fi.String("t2.medium"),
+				InstanceType:                 fi.String("t2.medium"),
-				SpotPrice:              "0.1",
+				SpotPrice:                    "0.1",
-				SpotDurationInMinutes:  fi.Int64(60),
+				SpotDurationInMinutes:        fi.Int64(60),
-				RootVolumeOptimization: fi.Bool(true),
+				InstanceInterruptionBehavior: fi.String("hibernate"),
-				RootVolumeIops:         fi.Int64(100),
+				RootVolumeOptimization:       fi.Bool(true),
-				RootVolumeSize:         fi.Int64(64),
+				RootVolumeIops:               fi.Int64(100),
+				RootVolumeSize:               fi.Int64(64),
 				SSHKey: &SSHKey{
 					Name:      fi.String("newkey"),
 					PublicKey: fi.WrapResource(fi.NewStringResource("newkey")),
@@ -61,8 +62,9 @@ resource "aws_launch_template" "test" {
   instance_market_options {
     market_type = "spot"
     spot_options {
-      block_duration_minutes = 60
+      block_duration_minutes         = 60
-      max_price              = "0.1"
+      instance_interruption_behavior = "hibernate"
+      max_price                      = "0.1"
     }
   }
   instance_type = "t2.medium"