Add ability to enable/configure warm pool for ASG

Apply suggestions from code review

Co-authored-by: John Gardiner Myers <jgmyers@proofpoint.com>

Apply suggestions from code review

Co-authored-by: John Gardiner Myers <jgmyers@proofpoint.com>

cloudmock/aws/mockautoscaling/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "ec2shim.go",
         "group.go",
         "tags.go",
+        "warmpool.go",
     ],
     importpath = "k8s.io/kops/cloudmock/aws/mockautoscaling",
     visibility = ["//visibility:public"],

cloudmock/aws/mockautoscaling/warmpool.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mockautoscaling
+
+import "github.com/aws/aws-sdk-go/service/autoscaling"
+
+func (m *MockAutoscaling) DescribeWarmPool(input *autoscaling.DescribeWarmPoolInput) (*autoscaling.DescribeWarmPoolOutput, error) {
+	instances, found := m.WarmPoolInstances[*input.AutoScalingGroupName]
+	if !found {
+		return &autoscaling.DescribeWarmPoolOutput{}, nil
+	}
+	ret := &autoscaling.DescribeWarmPoolOutput{
+		Instances: instances,
+	}
+	return ret, nil
+}

k8s/crds/kops.k8s.io_instancegroups.yaml

@@ -902,6 +902,23 @@ spec:
                       type: string
                   type: object
                 type: array
+              warmPool:
+                description: WarmPool configures an ASG warm pool for the instance
+                  group
+                properties:
+                  maxSize:
+                    description: MaxSize is the maximum size of the warm pool. The
+                      desired size of the instance group is subtracted from this number
+                      to determine the desired size of the warm pool (unless the resulting
+                      number is smaller than MinSize). The default is the instance
+                      group's MaxSize.
+                    format: int64
+                    type: integer
+                  minSize:
+                    description: MinSize is the minimum size of the pool
+                    format: int64
+                    type: integer
+                type: object
               zones:
                 description: Zones is the names of the Zones where machines in this
                   instance group should be placed This is needed for regional subnets

pkg/apis/kops/instancegroup.go

@@ -182,6 +182,18 @@ type InstanceGroupSpec struct {
 	//   'automatic' (default): apply updates automatically (apply OS security upgrades, avoiding rebooting when possible)
 	//   'external': do not apply updates automatically; they are applied manually or by an external system
 	UpdatePolicy *string `json:"updatePolicy,omitempty"`
+	// WarmPool specifies a pool of pre-warmed instances for later use (AWS only).
+	WarmPool *WarmPoolSpec `json:"warmPool,omitempty"`
+}
+
+type WarmPoolSpec struct {
+	// MinSize is the minimum size of the warm pool.
+	MinSize int64 `json:"minSize,omitempty"`
+	// MaxSize is the maximum size of the warm pool. The desired size of the instance group
+	// is subtracted from this number to determine the desired size of the warm pool
+	// (unless the resulting number is smaller than MinSize).
+	// The default is the instance group's MaxSize.
+	MaxSize *int64 `json:"maxSize,omitempty"`
 }
 
 const (

pkg/apis/kops/v1alpha2/instancegroup.go

@@ -148,6 +148,17 @@ type InstanceGroupSpec struct {
 	//   'automatic' (default): apply updates automatically (apply OS security upgrades, avoiding rebooting when possible)
 	//   'external': do not apply updates automatically; they are applied manually or by an external system
 	UpdatePolicy *string `json:"updatePolicy,omitempty"`
+	// WarmPool configures an ASG warm pool for the instance group
+	WarmPool *WarmPoolSpec `json:"warmPool,omitempty"`
+}
+type WarmPoolSpec struct {
+	// MinSize is the minimum size of the pool
+	MinSize int64 `json:"minSize,omitempty"`
+	// MaxSize is the maximum size of the warm pool. The desired size of the instance group
+	// is subtracted from this number to determine the desired size of the warm pool
+	// (unless the resulting number is smaller than MinSize).
+	// The default is the instance group's MaxSize.
+	MaxSize *int64 `json:"maxSize,omitempty"`
 }
 
 // InstanceMetadataOptions defines the EC2 instance metadata service options (AWS Only)

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1043,6 +1043,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*WarmPoolSpec)(nil), (*kops.WarmPoolSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_WarmPoolSpec_To_kops_WarmPoolSpec(a.(*WarmPoolSpec), b.(*kops.WarmPoolSpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.WarmPoolSpec)(nil), (*WarmPoolSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_WarmPoolSpec_To_v1alpha2_WarmPoolSpec(a.(*kops.WarmPoolSpec), b.(*WarmPoolSpec), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*WeaveNetworkingSpec)(nil), (*kops.WeaveNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_WeaveNetworkingSpec_To_kops_WeaveNetworkingSpec(a.(*WeaveNetworkingSpec), b.(*kops.WeaveNetworkingSpec), scope)
 	}); err != nil {
@@ -3987,6 +3997,15 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 		out.InstanceMetadata = nil
 	}
 	out.UpdatePolicy = in.UpdatePolicy
+	if in.WarmPool != nil {
+		in, out := &in.WarmPool, &out.WarmPool
+		*out = new(kops.WarmPoolSpec)
+		if err := Convert_v1alpha2_WarmPoolSpec_To_kops_WarmPoolSpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.WarmPool = nil
+	}
 	return nil
 }
 
@@ -4140,6 +4159,15 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
 		out.InstanceMetadata = nil
 	}
 	out.UpdatePolicy = in.UpdatePolicy
+	if in.WarmPool != nil {
+		in, out := &in.WarmPool, &out.WarmPool
+		*out = new(WarmPoolSpec)
+		if err := Convert_kops_WarmPoolSpec_To_v1alpha2_WarmPoolSpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.WarmPool = nil
+	}
 	return nil
 }
 
@@ -6372,6 +6400,28 @@ func Convert_kops_VolumeSpec_To_v1alpha2_VolumeSpec(in *kops.VolumeSpec, out *Vo
 	return autoConvert_kops_VolumeSpec_To_v1alpha2_VolumeSpec(in, out, s)
 }
 
+func autoConvert_v1alpha2_WarmPoolSpec_To_kops_WarmPoolSpec(in *WarmPoolSpec, out *kops.WarmPoolSpec, s conversion.Scope) error {
+	out.MinSize = in.MinSize
+	out.MaxSize = in.MaxSize
+	return nil
+}
+
+// Convert_v1alpha2_WarmPoolSpec_To_kops_WarmPoolSpec is an autogenerated conversion function.
+func Convert_v1alpha2_WarmPoolSpec_To_kops_WarmPoolSpec(in *WarmPoolSpec, out *kops.WarmPoolSpec, s conversion.Scope) error {
+	return autoConvert_v1alpha2_WarmPoolSpec_To_kops_WarmPoolSpec(in, out, s)
+}
+
+func autoConvert_kops_WarmPoolSpec_To_v1alpha2_WarmPoolSpec(in *kops.WarmPoolSpec, out *WarmPoolSpec, s conversion.Scope) error {
+	out.MinSize = in.MinSize
+	out.MaxSize = in.MaxSize
+	return nil
+}
+
+// Convert_kops_WarmPoolSpec_To_v1alpha2_WarmPoolSpec is an autogenerated conversion function.
+func Convert_kops_WarmPoolSpec_To_v1alpha2_WarmPoolSpec(in *kops.WarmPoolSpec, out *WarmPoolSpec, s conversion.Scope) error {
+	return autoConvert_kops_WarmPoolSpec_To_v1alpha2_WarmPoolSpec(in, out, s)
+}
+
 func autoConvert_v1alpha2_WeaveNetworkingSpec_To_kops_WeaveNetworkingSpec(in *WeaveNetworkingSpec, out *kops.WeaveNetworkingSpec, s conversion.Scope) error {
 	out.MTU = in.MTU
 	out.ConnLimit = in.ConnLimit

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -2194,6 +2194,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.WarmPool != nil {
+		in, out := &in.WarmPool, &out.WarmPool
+		*out = new(WarmPoolSpec)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -4467,6 +4472,27 @@ func (in *VolumeSpec) DeepCopy() *VolumeSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WarmPoolSpec) DeepCopyInto(out *WarmPoolSpec) {
+	*out = *in
+	if in.MaxSize != nil {
+		in, out := &in.MaxSize, &out.MaxSize
+		*out = new(int64)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WarmPoolSpec.
+func (in *WarmPoolSpec) DeepCopy() *WarmPoolSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(WarmPoolSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WeaveNetworkingSpec) DeepCopyInto(out *WeaveNetworkingSpec) {
 	*out = *in

pkg/apis/kops/validation/instancegroup.go

@@ -143,6 +143,24 @@ func ValidateInstanceGroup(g *kops.InstanceGroup, cloud fi.Cloud) field.ErrorLis
 
 	allErrs = append(allErrs, IsValidValue(field.NewPath("spec", "updatePolicy"), g.Spec.UpdatePolicy, []string{kops.UpdatePolicyAutomatic, kops.UpdatePolicyExternal})...)
 
+	warmPool := g.Spec.WarmPool
+	if warmPool != nil {
+		if g.Spec.Role != kops.InstanceGroupRoleNode && g.Spec.Role != kops.InstanceGroupRoleAPIServer {
+			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "warmPool"), "warm pool only allowed on instance groups with role Node or APIServer"))
+		}
+		if g.Spec.MixedInstancesPolicy != nil {
+			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "warmPool"), "warm pool cannot be combined with a mixed instances policy"))
+		}
+		if g.Spec.MaxPrice != nil {
+			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "warmPool"), "warm pool cannot be used with spot instances"))
+		}
+		if warmPool.MaxSize != nil {
+			if warmPool.MinSize > *warmPool.MaxSize {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "warmPool", "maxSize"), fi.Int64Value(warmPool.MaxSize), "warm pool maxSize cannot be set to lower than minSize"))
+
+			}
+		}
+	}
 	return allErrs
 }
 
@@ -206,10 +224,15 @@ func CrossValidateInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster, cl
 		}
 	}
 
-	if g.Spec.RootVolumeType != nil && kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS {
-		allErrs = append(allErrs, IsValidValue(field.NewPath("spec", "rootVolumeType"), g.Spec.RootVolumeType, []string{"standard", "gp3", "gp2", "io1", "io2"})...)
+	if kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS {
+		if g.Spec.RootVolumeType != nil {
+			allErrs = append(allErrs, IsValidValue(field.NewPath("spec", "rootVolumeType"), g.Spec.RootVolumeType, []string{"standard", "gp3", "gp2", "io1", "io2"})...)
+		}
+	} else {
+		if g.Spec.WarmPool != nil {
+			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "warmPool"), "warm pool only supported on AWS"))
+		}
 	}
-
 	return allErrs
 }
 

pkg/apis/kops/zz_generated.deepcopy.go

@@ -2360,6 +2360,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.WarmPool != nil {
+		in, out := &in.WarmPool, &out.WarmPool
+		*out = new(WarmPoolSpec)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -4681,6 +4686,27 @@ func (in *VolumeSpec) DeepCopy() *VolumeSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WarmPoolSpec) DeepCopyInto(out *WarmPoolSpec) {
+	*out = *in
+	if in.MaxSize != nil {
+		in, out := &in.MaxSize, &out.MaxSize
+		*out = new(int64)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WarmPoolSpec.
+func (in *WarmPoolSpec) DeepCopy() *WarmPoolSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(WarmPoolSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WeaveNetworkingSpec) DeepCopyInto(out *WeaveNetworkingSpec) {
 	*out = *in

pkg/model/awsmodel/autoscalinggroup.go

@@ -85,6 +85,20 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 		tsk.LaunchTemplate = task
 		c.AddTask(tsk)
+
+		warmPool := ig.Spec.WarmPool
+		{
+			enabled := fi.Bool(warmPool != nil)
+			warmPoolTask := &awstasks.WarmPool{
+				Name:    &name,
+				Enabled: enabled,
+			}
+			if warmPool != nil {
+				warmPoolTask.MinSize = warmPool.MinSize
+				warmPoolTask.MaxSize = warmPool.MaxSize
+			}
+			c.AddTask(warmPoolTask)
+		}
 	}
 
 	return nil

upup/pkg/fi/cloudup/awstasks/BUILD.bazel

@@ -73,6 +73,8 @@ go_library(
         "vpccidrblock.go",
         "vpccidrblock_fitask.go",
         "vpcdhcpoptionsassociation_fitask.go",
+        "warmpool.go",
+        "warmpool_fitask.go",
     ],
     importpath = "k8s.io/kops/upup/pkg/fi/cloudup/awstasks",
     visibility = ["//visibility:public"],

upup/pkg/fi/cloudup/awstasks/warmpool.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awstasks
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"k8s.io/klog/v2"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
+	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
+)
+
+// WarmPool provdes the definition for an ASG warm pool in aws.
+// +kops:fitask
+type WarmPool struct {
+	// Name is the name of the ASG.
+	Name *string
+	// Lifecycle is the resource lifecycle.
+	Lifecycle *fi.Lifecycle
+
+	Enabled *bool
+	// MaxSize is the max number of nodes in the warm pool.
+	MaxSize *int64
+	// MinSize is the smallest number of nodes in the warm pool.
+	MinSize int64
+
+	AutoscalingGroup *AutoscalingGroup
+}
+
+// Find is used to discover the ASG in the cloud provider.
+func (e *WarmPool) Find(c *fi.Context) (*WarmPool, error) {
+	cloud := c.Cloud.(awsup.AWSCloud)
+	svc := cloud.Autoscaling()
+	warmPool, err := svc.DescribeWarmPool(&autoscaling.DescribeWarmPoolInput{
+		AutoScalingGroupName: e.Name,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if warmPool.WarmPoolConfiguration == nil {
+		return &WarmPool{
+			Name:    e.Name,
+			Enabled: fi.Bool(false),
+		}, nil
+	}
+
+	actual := &WarmPool{
+		Name:    e.Name,
+		Enabled: fi.Bool(true),
+		MaxSize: warmPool.WarmPoolConfiguration.MaxGroupPreparedCapacity,
+		MinSize: fi.Int64Value(warmPool.WarmPoolConfiguration.MinSize),
+	}
+	return actual, nil
+}
+
+func (e *WarmPool) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(e, c)
+}
+
+func (*WarmPool) CheckChanges(a, e, changes *WarmPool) error {
+	return nil
+}
+
+func (*WarmPool) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *WarmPool) error {
+
+	svc := t.Cloud.Autoscaling()
+	if changes != nil {
+		if fi.BoolValue(e.Enabled) {
+			minSize := e.MinSize
+			maxSize := e.MaxSize
+			if maxSize == nil {
+				maxSize = fi.Int64(-1)
+			}
+			request := &autoscaling.PutWarmPoolInput{
+				AutoScalingGroupName:     e.Name,
+				MaxGroupPreparedCapacity: maxSize,
+				MinSize:                  fi.Int64(minSize),
+			}
+
+			_, err := svc.PutWarmPool(request)
+			if err != nil {
+				return fmt.Errorf("error modifying warm pool: %w", err)
+			}
+		} else {
+			_, err := svc.DeleteWarmPool(&autoscaling.DeleteWarmPoolInput{
+				AutoScalingGroupName: e.Name,
+				// We don't need to do any cleanup so, the faster the better
+				ForceDelete: fi.Bool(true),
+			})
+			if err != nil {
+				return fmt.Errorf("error modifying warm pool: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func (_ *WarmPool) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *WarmPool) error {
+	if changes != nil {
+		klog.Warning("ASG warm pool is not supported by the terraform target")
+	}
+	return nil
+}
+
+func (_ *WarmPool) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *WarmPool) error {
+	if changes != nil {
+		klog.Warning("ASG warm pool is not supported by the cloudformation target")
+	}
+	return nil
+}

upup/pkg/fi/cloudup/awstasks/warmpool_fitask.go

@@ -0,0 +1,51 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by fitask. DO NOT EDIT.
+
+package awstasks
+
+import (
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// WarmPool
+
+var _ fi.HasLifecycle = &WarmPool{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *WarmPool) GetLifecycle() *fi.Lifecycle {
+	return o.Lifecycle
+}
+
+// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
+func (o *WarmPool) SetLifecycle(lifecycle fi.Lifecycle) {
+	o.Lifecycle = &lifecycle
+}
+
+var _ fi.HasName = &WarmPool{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *WarmPool) GetName() *string {
+	return o.Name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *WarmPool) String() string {
+	return fi.TaskAsString(o)
+}