Merge pull request #9221 from spotinst/feat-draining-timeout

Spotinst: Allow a user specifiable node draining timeout

docs/getting_started/spot-ocean.md

@@ -152,6 +152,7 @@ metadata:
 | `spotinst.io/spot-percentage` | Specify the percentage of Spot instances that should spin up from the target capacity. | `100` |
 | `spotinst.io/utilize-reserved-instances` | Specify whether reserved instances should be utilized. | `true` |
 | `spotinst.io/fallback-to-ondemand` | Specify whether fallback to on-demand instances should be enabled. | `true` |
+| `spotinst.io/draining-timeout` | Specify a period of time, in seconds, after a node is marked for termination during which on running pods remains active. | none |
 | `spotinst.io/grace-period` | Specify a period of time, in seconds, that Ocean should wait before applying instance health checks. | none |
 | `spotinst.io/ocean-default-launchspec` | Specify whether to use the InstanceGroup's spec as the default Launch Spec for the Ocean cluster. | none |
 | `spotinst.io/ocean-instance-types-whitelist` | Specify whether to whitelist specific instance types. | none |

pkg/model/spotinstmodel/instance_group.go

@@ -57,6 +57,11 @@ const (
 	// be enabled.
 	InstanceGroupLabelFallbackToOnDemand = "spotinst.io/fallback-to-ondemand"
 
+	// InstanceGroupLabelDrainingTimeout is the metadata label used on the
+	// instance group to specify a period of time, in seconds, after a node
+	// is marked for termination during which on running pods remains active.
+	InstanceGroupLabelDrainingTimeout = "spotinst.io/draining-timeout"
+
 	// InstanceGroupLabelGracePeriod is the metadata label used on the
 	// instance group to specify a period of time, in seconds, that Ocean
 	// should wait before applying instance health checks.
@@ -202,6 +207,12 @@ func (b *InstanceGroupModelBuilder) buildElastigroup(c *fi.ModelBuilderContext,
 				return err
 			}
 
+		case InstanceGroupLabelDrainingTimeout:
+			group.DrainingTimeout, err = parseInt(v)
+			if err != nil {
+				return err
+			}
+
 		case InstanceGroupLabelHealthCheckType:
 			group.HealthCheckType = fi.String(strings.ToUpper(v))
 		}
@@ -378,6 +389,12 @@ func (b *InstanceGroupModelBuilder) buildOcean(c *fi.ModelBuilderContext, igs ..
 				return err
 			}
 
+		case InstanceGroupLabelDrainingTimeout:
+			ocean.DrainingTimeout, err = parseInt(v)
+			if err != nil {
+				return err
+			}
+
 		case InstanceGroupLabelOceanInstanceTypesWhitelist:
 			ocean.InstanceTypesWhitelist, err = parseStringSlice(v)
 			if err != nil {

upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go

@@ -50,6 +50,7 @@ type Elastigroup struct {
 	SpotPercentage           *float64
 	UtilizeReservedInstances *bool
 	FallbackToOnDemand       *bool
+	DrainingTimeout          *int64
 	HealthCheckType          *string
 	Product                  *string
 	Orientation              *string
@@ -186,6 +187,10 @@ func (e *Elastigroup) Find(c *fi.Context) (*Elastigroup, error) {
 		actual.Orientation = group.Strategy.AvailabilityVsCost
 		actual.FallbackToOnDemand = group.Strategy.FallbackToOnDemand
 		actual.UtilizeReservedInstances = group.Strategy.UtilizeReservedInstances
+
+		if group.Strategy.DrainingTimeout != nil {
+			actual.DrainingTimeout = fi.Int64(int64(fi.IntValue(group.Strategy.DrainingTimeout)))
+		}
 	}
 
 	// Compute.
@@ -479,6 +484,10 @@ func (_ *Elastigroup) create(cloud awsup.AWSCloud, a, e, changes *Elastigroup) e
 		group.Strategy.SetAvailabilityVsCost(fi.String(string(normalizeOrientation(e.Orientation))))
 		group.Strategy.SetFallbackToOnDemand(e.FallbackToOnDemand)
 		group.Strategy.SetUtilizeReservedInstances(e.UtilizeReservedInstances)
+
+		if e.DrainingTimeout != nil {
+			group.Strategy.SetDrainingTimeout(fi.Int(int(*e.DrainingTimeout)))
+		}
 	}
 
 	// Compute.
@@ -782,6 +791,17 @@ func (_ *Elastigroup) update(cloud awsup.AWSCloud, a, e, changes *Elastigroup) e
 			changes.UtilizeReservedInstances = nil
 			changed = true
 		}
+
+		// Draining timeout.
+		if changes.DrainingTimeout != nil {
+			if group.Strategy == nil {
+				group.Strategy = new(aws.Strategy)
+			}
+
+			group.Strategy.SetDrainingTimeout(fi.Int(int(*e.DrainingTimeout)))
+			changes.DrainingTimeout = nil
+			changed = true
+		}
 	}
 
 	// Compute.
@@ -1275,6 +1295,7 @@ type terraformElastigroupStrategy struct {
 	Orientation              *string  `json:"orientation,omitempty" cty:"orientation"`
 	FallbackToOnDemand       *bool    `json:"fallback_to_ondemand,omitempty" cty:"fallback_to_ondemand"`
 	UtilizeReservedInstances *bool    `json:"utilize_reserved_instances,omitempty" cty:"utilize_reserved_instances"`
+	DrainingTimeout          *int64   `json:"draining_timeout,omitempty" cty:"draining_timeout"`
 }
 
 type terraformElastigroupInstanceTypes struct {
@@ -1365,6 +1386,7 @@ func (_ *Elastigroup) RenderTerraform(t *terraform.TerraformTarget, a, e, change
 			Orientation:              fi.String(string(normalizeOrientation(e.Orientation))),
 			FallbackToOnDemand:       e.FallbackToOnDemand,
 			UtilizeReservedInstances: e.UtilizeReservedInstances,
+			DrainingTimeout:          e.DrainingTimeout,
 		},
 		terraformElastigroupInstanceTypes: &terraformElastigroupInstanceTypes{
 			OnDemand: e.OnDemandInstanceType,

upup/pkg/fi/cloudup/spotinsttasks/ocean.go

@@ -47,6 +47,7 @@ type Ocean struct {
 	SpotPercentage           *float64
 	UtilizeReservedInstances *bool
 	FallbackToOnDemand       *bool
+	DrainingTimeout          *int64
 	GracePeriod              *int64
 	InstanceTypesWhitelist   []string
 	InstanceTypesBlacklist   []string
@@ -147,6 +148,10 @@ func (o *Ocean) Find(c *fi.Context) (*Ocean, error) {
 			actual.FallbackToOnDemand = strategy.FallbackToOnDemand
 			actual.UtilizeReservedInstances = strategy.UtilizeReservedInstances
 
+			if strategy.DrainingTimeout != nil {
+				actual.DrainingTimeout = fi.Int64(int64(fi.IntValue(strategy.DrainingTimeout)))
+			}
+
 			if strategy.GracePeriod != nil {
 				actual.GracePeriod = fi.Int64(int64(fi.IntValue(strategy.GracePeriod)))
 			}
@@ -372,6 +377,10 @@ func (_ *Ocean) create(cloud awsup.AWSCloud, a, e, changes *Ocean) error {
 		ocean.Strategy.SetFallbackToOnDemand(e.FallbackToOnDemand)
 		ocean.Strategy.SetUtilizeReservedInstances(e.UtilizeReservedInstances)
 
+		if e.DrainingTimeout != nil {
+			ocean.Strategy.SetDrainingTimeout(fi.Int(int(*e.DrainingTimeout)))
+		}
+
 		if e.GracePeriod != nil {
 			ocean.Strategy.SetGracePeriod(fi.Int(int(*e.GracePeriod)))
 		}
@@ -620,6 +629,17 @@ func (_ *Ocean) update(cloud awsup.AWSCloud, a, e, changes *Ocean) error {
 			changed = true
 		}
 
+		// Draining timeout.
+		if changes.DrainingTimeout != nil {
+			if ocean.Strategy == nil {
+				ocean.Strategy = new(aws.Strategy)
+			}
+
+			ocean.Strategy.SetDrainingTimeout(fi.Int(int(*e.DrainingTimeout)))
+			changes.DrainingTimeout = nil
+			changed = true
+		}
+
 		// Grace period.
 		if changes.GracePeriod != nil {
 			if ocean.Strategy == nil {
@@ -997,6 +1017,7 @@ type terraformOceanStrategy struct {
 	SpotPercentage           *float64 `json:"spot_percentage,omitempty" cty:"spot_percentage"`
 	FallbackToOnDemand       *bool    `json:"fallback_to_ondemand,omitempty" cty:"fallback_to_ondemand"`
 	UtilizeReservedInstances *bool    `json:"utilize_reserved_instances,omitempty" cty:"utilize_reserved_instances"`
+	DrainingTimeout          *int64   `json:"draining_timeout,omitempty" cty:"draining_timeout"`
 	GracePeriod              *int64   `json:"grace_period,omitempty" cty:"grace_period"`
 }
 
@@ -1033,6 +1054,7 @@ func (_ *Ocean) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Oce
 			SpotPercentage:           e.SpotPercentage,
 			FallbackToOnDemand:       e.FallbackToOnDemand,
 			UtilizeReservedInstances: e.UtilizeReservedInstances,
+			DrainingTimeout:          e.DrainingTimeout,
 			GracePeriod:              e.GracePeriod,
 		},
 		terraformOceanLaunchSpec: &terraformOceanLaunchSpec{},