Add suspendProcesses to IGs

docs/instance_groups.md

@@ -219,7 +219,7 @@ spec:
 Kops utilizes cloud-init to initialize and setup a host at boot time. However in certain cases you may already be leaveraging certain features of cloud-init in your infrastructure and would like to continue doing so. More information on cloud-init can be found [here](http://cloudinit.readthedocs.io/en/latest/)
 
 
-Aditional user-user data can be passed to the host provisioning by setting the `AdditionalUserData` field. A list of valid user-data content-types can be found [here](http://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive) 
+Aditional user-user data can be passed to the host provisioning by setting the `AdditionalUserData` field. A list of valid user-data content-types can be found [here](http://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive)
 
 Example:
 ```
@@ -252,7 +252,7 @@ If you need to add tags on auto scaling groups or instances (propagate ASG tags)
 apiVersion: kops/v1alpha2
 kind: InstanceGroup
 metadata:
-  labels: 
+  labels:
     kops.k8s.io/cluster: k8s.dev.local
   name: nodes
 spec:
@@ -265,3 +265,29 @@ spec:
   minSize: 2
   role: Node
 ```
+
+## Suspending Scaling Processes on AWS Autoscaling groups
+
+Autoscaling groups automatically include multiple [scaling processes](https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-suspend-resume-processes.html#process-types)
+that keep our ASGs healthy.  In some cases, you may want to disable certain scaling activities.
+
+An example of this is if you are running multiple AZs in an ASG while using a Kubernetes Autoscaler.
+The autoscaler will remove specific instances that are not being used.  In some cases, the `AZRebalance` process
+will rescale the ASG without warning.
+
+```
+# Example for nodes
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  labels:
+    kops.k8s.io/cluster: k8s.dev.local
+  name: nodes
+spec:
+  machineType: m4.xlarge
+  maxSize: 20
+  minSize: 2
+  role: Node
+  suspendProcesses:
+  - AZRebalance
+```

pkg/apis/kops/instancegroup.go

@@ -114,6 +114,8 @@ type InstanceGroupSpec struct {
 	Taints []string `json:"taints,omitempty"`
 	// AdditionalUserData is any aditional user-data to be passed to the host
 	AdditionalUserData []UserData `json:"additionalUserData,omitempty"`
+	// SuspendProcesses disables the listed Scaling Policies
+	SuspendProcesses []string `json:"suspendProcesses,omitempty"`
 }
 
 // UserData defines a user-data section

pkg/apis/kops/v1alpha1/instancegroup.go

@@ -94,6 +94,8 @@ type InstanceGroupSpec struct {
 	// Zones is the names of the Zones where machines in this instance group should be placed
 	// This is needed for regional subnets (e.g. GCE), to restrict placement to particular zones
 	Zones []string `json:"zones,omitempty"`
+	// SuspendProcesses disables the listed Scaling Policies
+	SuspendProcesses []string `json:"suspendProcesses,omitempty"`
 }
 
 // UserData defines a user-data section

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -1647,6 +1647,7 @@ func autoConvert_v1alpha1_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 		out.AdditionalUserData = nil
 	}
 	out.Zones = in.Zones
+	out.SuspendProcesses = in.SuspendProcesses
 	return nil
 }
 
@@ -1711,6 +1712,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha1_InstanceGroupSpec(in *kops.I
 	} else {
 		out.AdditionalUserData = nil
 	}
+	out.SuspendProcesses = in.SuspendProcesses
 	return nil
 }
 

pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go

@@ -1502,6 +1502,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.SuspendProcesses != nil {
+		in, out := &in.SuspendProcesses, &out.SuspendProcesses
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

pkg/apis/kops/v1alpha2/instancegroup.go

@@ -103,6 +103,8 @@ type InstanceGroupSpec struct {
 	Taints []string `json:"taints,omitempty"`
 	// AdditionalUserData is any aditional user-data to be passed to the host
 	AdditionalUserData []UserData `json:"additionalUserData,omitempty"`
+	// SuspendProcesses disables the listed Scaling Policies
+	SuspendProcesses []string `json:"suspendProcesses,omitempty"`
 }
 
 // UserData defines a user-data section

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1757,6 +1757,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 	} else {
 		out.AdditionalUserData = nil
 	}
+	out.SuspendProcesses = in.SuspendProcesses
 	return nil
 }
 
@@ -1826,6 +1827,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
 	} else {
 		out.AdditionalUserData = nil
 	}
+	out.SuspendProcesses = in.SuspendProcesses
 	return nil
 }
 

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -1501,6 +1501,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = make([]UserData, len(*in))
 		copy(*out, *in)
 	}
+	if in.SuspendProcesses != nil {
+		in, out := &in.SuspendProcesses, &out.SuspendProcesses
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

pkg/apis/kops/zz_generated.deepcopy.go

@@ -1664,6 +1664,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = make([]UserData, len(*in))
 		copy(*out, *in)
 	}
+	if in.SuspendProcesses != nil {
+		in, out := &in.SuspendProcesses, &out.SuspendProcesses
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

pkg/model/awsmodel/autoscalinggroup.go

@@ -224,6 +224,12 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			}
 			t.Tags = tags
 
+			if ig.Spec.SuspendProcesses != nil {
+				for _, p := range ig.Spec.SuspendProcesses {
+					t.SuspendProcesses = append(t.SuspendProcesses, p)
+				}
+			}
+
 			c.AddTask(t)
 		}
 	}

pkg/model/bootstrapscript.go

@@ -149,6 +149,7 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste
 			spec["kubelet"] = ig.Spec.Kubelet
 			spec["nodeLabels"] = ig.Spec.NodeLabels
 			spec["taints"] = ig.Spec.Taints
+			spec["suspendProcesses"] = ig.Spec.SuspendProcesses
 
 			hooks, err := b.getRelevantHooks(ig.Spec.Hooks, ig.Spec.Role)
 			if err != nil {

pkg/model/bootstrapscript_test.go

@@ -261,6 +261,9 @@ func makeTestInstanceGroup(role kops.InstanceGroupRole, hookSpecRoles []kops.Ins
 				"key1=value1:NoSchedule",
 				"key2=value2:NoExecute",
 			},
+			SuspendProcesses: []string{
+				"AZRebalance",
+			},
 			Hooks: []kops.HookSpec{
 				{
 					Name: "disable-update-engine.service",

pkg/model/tests/data/bootstrapscript_0.txt

@@ -195,6 +195,8 @@ kubelet:
 nodeLabels:
   label2: value2
   labelname: labelvalue
+suspendProcesses:
+- AZRebalance
 taints:
 - key1=value1:NoSchedule
 - key2=value2:NoExecute

pkg/model/tests/data/bootstrapscript_1.txt

@@ -212,6 +212,8 @@ kubelet:
 nodeLabels:
   label2: value2
   labelname: labelvalue
+suspendProcesses:
+- AZRebalance
 taints:
 - key1=value1:NoSchedule
 - key2=value2:NoExecute

pkg/model/tests/data/bootstrapscript_2.txt

@@ -212,6 +212,8 @@ kubelet:
 nodeLabels:
   label2: value2
   labelname: labelvalue
+suspendProcesses:
+- AZRebalance
 taints:
 - key1=value1:NoSchedule
 - key2=value2:NoExecute

pkg/model/tests/data/bootstrapscript_3.txt

@@ -180,6 +180,8 @@ kubelet:
 nodeLabels:
   label2: value2
   labelname: labelvalue
+suspendProcesses:
+- AZRebalance
 taints:
 - key1=value1:NoSchedule
 - key2=value2:NoExecute

pkg/model/tests/data/bootstrapscript_4.txt

@@ -197,6 +197,8 @@ kubelet:
 nodeLabels:
   label2: value2
   labelname: labelvalue
+suspendProcesses:
+- AZRebalance
 taints:
 - key1=value1:NoSchedule
 - key2=value2:NoExecute

pkg/model/tests/data/bootstrapscript_5.txt

@@ -197,6 +197,8 @@ kubelet:
 nodeLabels:
   label2: value2
   labelname: labelvalue
+suspendProcesses:
+- AZRebalance
 taints:
 - key1=value1:NoSchedule
 - key2=value2:NoExecute

tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml

@@ -254,6 +254,7 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersadditionaluserda
   cat > ig_spec.yaml << '__EOF_IG_SPEC'
   kubelet: null
   nodeLabels: null
+  suspendProcesses: null
   taints: null
 
   __EOF_IG_SPEC
@@ -483,6 +484,7 @@ Resources.AWSAutoScalingLaunchConfigurationnodesadditionaluserdataexamplecom.Pro
   cat > ig_spec.yaml << '__EOF_IG_SPEC'
   kubelet: null
   nodeLabels: null
+  suspendProcesses: null
   taints: null
 
   __EOF_IG_SPEC

tests/integration/update_cluster/complex/in-v1alpha2.yaml

@@ -64,6 +64,8 @@ spec:
   - sg-exampleid3
   - sg-exampleid4
   associatePublicIp: true
+  suspendProcesses:
+  - AZRebalance
   image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
   machineType: t2.medium
   maxSize: 2

tests/integration/update_cluster/complex/kubernetes.tf

@@ -127,6 +127,7 @@ resource "aws_autoscaling_group" "nodes-complex-example-com" {
 
   metrics_granularity = "1Minute"
   enabled_metrics     = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+  suspended_processes = ["AZRebalance"]
 }
 
 resource "aws_ebs_volume" "us-test-1a-etcd-events-complex-example-com" {

tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml

@@ -245,6 +245,7 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersminimalexampleco
   cat > ig_spec.yaml << '__EOF_IG_SPEC'
   kubelet: null
   nodeLabels: null
+  suspendProcesses: null
   taints: null
 
   __EOF_IG_SPEC
@@ -453,6 +454,7 @@ Resources.AWSAutoScalingLaunchConfigurationnodesminimalexamplecom.Properties.Use
   cat > ig_spec.yaml << '__EOF_IG_SPEC'
   kubelet: null
   nodeLabels: null
+  suspendProcesses: null
   taints: null
 
   __EOF_IG_SPEC

upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go

@@ -48,6 +48,8 @@ type AutoscalingGroup struct {
 	Metrics     []string
 
 	LaunchConfiguration *LaunchConfiguration
+
+	SuspendProcesses []string
 }
 
 var _ fi.CompareWithID = &AutoscalingGroup{}
@@ -304,6 +306,21 @@ func (_ *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos
 		}
 	}
 
+	if e.SuspendProcesses != nil {
+		processQuery := &autoscaling.ScalingProcessQuery{}
+		processQuery.AutoScalingGroupName = e.Name
+		processQuery.ScalingProcesses = []*string{}
+
+		for _, p := range e.SuspendProcesses {
+			processQuery.ScalingProcesses = append(processQuery.ScalingProcesses, &p)
+		}
+
+		_, err := t.Cloud.Autoscaling().SuspendProcesses(processQuery)
+		if err != nil {
+			return fmt.Errorf("error suspending processes: %v", err)
+		}
+	}
+
 	// TODO: Use PropagateAtLaunch = false for tagging?
 
 	return nil // We have
@@ -341,9 +358,11 @@ type terraformAutoscalingGroup struct {
 	Tags                    []*terraformASGTag   `json:"tag,omitempty"`
 	MetricsGranularity      *string              `json:"metrics_granularity,omitempty"`
 	EnabledMetrics          []*string            `json:"enabled_metrics,omitempty"`
+	SuspendedProcesses      []*string            `json:"suspended_processes, omitempty"`
 }
 
 func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *AutoscalingGroup) error {
+
 	tf := &terraformAutoscalingGroup{
 		Name:                    e.Name,
 		MinSize:                 e.MinSize,
@@ -405,6 +424,14 @@ func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, c
 		}
 	}
 
+	var processes []*string
+	if e.SuspendProcesses != nil {
+		for _, p := range e.SuspendProcesses {
+			processes = append(processes, fi.String(p))
+		}
+	}
+	tf.SuspendedProcesses = processes
+
 	return t.RenderResource("aws_autoscaling_group", *e.Name, tf)
 }