diff --git a/docs/instance_groups.md b/docs/instance_groups.md index f5d4eeed51..7d10186353 100644 --- a/docs/instance_groups.md +++ b/docs/instance_groups.md @@ -219,7 +219,7 @@ spec: Kops utilizes cloud-init to initialize and setup a host at boot time. However in certain cases you may already be leaveraging certain features of cloud-init in your infrastructure and would like to continue doing so. More information on cloud-init can be found [here](http://cloudinit.readthedocs.io/en/latest/) -Aditional user-user data can be passed to the host provisioning by setting the `AdditionalUserData` field. A list of valid user-data content-types can be found [here](http://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive) +Aditional user-user data can be passed to the host provisioning by setting the `AdditionalUserData` field. A list of valid user-data content-types can be found [here](http://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive) Example: ``` @@ -252,7 +252,7 @@ If you need to add tags on auto scaling groups or instances (propagate ASG tags) apiVersion: kops/v1alpha2 kind: InstanceGroup metadata: - labels: + labels: kops.k8s.io/cluster: k8s.dev.local name: nodes spec: @@ -265,3 +265,29 @@ spec: minSize: 2 role: Node ``` + +## Suspending Scaling Processes on AWS Autoscaling groups + +Autoscaling groups automatically include multiple [scaling processes](https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-suspend-resume-processes.html#process-types) +that keep our ASGs healthy. In some cases, you may want to disable certain scaling activities. + +An example of this is if you are running multiple AZs in an ASG while using a Kubernetes Autoscaler. +The autoscaler will remove specific instances that are not being used. In some cases, the `AZRebalance` process +will rescale the ASG without warning. + +``` +# Example for nodes +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + labels: + kops.k8s.io/cluster: k8s.dev.local + name: nodes +spec: + machineType: m4.xlarge + maxSize: 20 + minSize: 2 + role: Node + suspendProcesses: + - AZRebalance +``` \ No newline at end of file diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 7a0cefc1f3..446e21f57d 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -114,6 +114,8 @@ type InstanceGroupSpec struct { Taints []string `json:"taints,omitempty"` // AdditionalUserData is any aditional user-data to be passed to the host AdditionalUserData []UserData `json:"additionalUserData,omitempty"` + // SuspendProcesses disables the listed Scaling Policies + SuspendProcesses []string `json:"suspendProcesses,omitempty"` } // UserData defines a user-data section diff --git a/pkg/apis/kops/v1alpha1/instancegroup.go b/pkg/apis/kops/v1alpha1/instancegroup.go index 6eef349954..05306e0b02 100644 --- a/pkg/apis/kops/v1alpha1/instancegroup.go +++ b/pkg/apis/kops/v1alpha1/instancegroup.go @@ -94,6 +94,8 @@ type InstanceGroupSpec struct { // Zones is the names of the Zones where machines in this instance group should be placed // This is needed for regional subnets (e.g. GCE), to restrict placement to particular zones Zones []string `json:"zones,omitempty"` + // SuspendProcesses disables the listed Scaling Policies + SuspendProcesses []string `json:"suspendProcesses,omitempty"` } // UserData defines a user-data section diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 554e4fd27e..aad4977a4a 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1691,6 +1691,7 @@ func autoConvert_v1alpha1_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan out.AdditionalUserData = nil } out.Zones = in.Zones + out.SuspendProcesses = in.SuspendProcesses return nil } @@ -1755,6 +1756,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha1_InstanceGroupSpec(in *kops.I } else { out.AdditionalUserData = nil } + out.SuspendProcesses = in.SuspendProcesses return nil } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index 2d17c08d11..f8d39af296 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -1532,6 +1532,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.SuspendProcesses != nil { + in, out := &in.SuspendProcesses, &out.SuspendProcesses + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index b8101301c9..d75511845b 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -103,6 +103,8 @@ type InstanceGroupSpec struct { Taints []string `json:"taints,omitempty"` // AdditionalUserData is any aditional user-data to be passed to the host AdditionalUserData []UserData `json:"additionalUserData,omitempty"` + // SuspendProcesses disables the listed Scaling Policies + SuspendProcesses []string `json:"suspendProcesses,omitempty"` } // UserData defines a user-data section diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 436ba648db..efa5c04bf5 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1801,6 +1801,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan } else { out.AdditionalUserData = nil } + out.SuspendProcesses = in.SuspendProcesses return nil } @@ -1870,6 +1871,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I } else { out.AdditionalUserData = nil } + out.SuspendProcesses = in.SuspendProcesses return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index f66d1ab197..884d223752 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1531,6 +1531,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = make([]UserData, len(*in)) copy(*out, *in) } + if in.SuspendProcesses != nil { + in, out := &in.SuspendProcesses, &out.SuspendProcesses + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 530b48015b..f4a8e4032e 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1694,6 +1694,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = make([]UserData, len(*in)) copy(*out, *in) } + if in.SuspendProcesses != nil { + in, out := &in.SuspendProcesses, &out.SuspendProcesses + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 22ca25458f..be3278d42a 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -224,6 +224,12 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { } t.Tags = tags + if ig.Spec.SuspendProcesses != nil { + for _, p := range ig.Spec.SuspendProcesses { + t.SuspendProcesses = append(t.SuspendProcesses, p) + } + } + c.AddTask(t) } } diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 65bbd0b8ff..fdc480e9c0 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -149,6 +149,7 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste spec["kubelet"] = ig.Spec.Kubelet spec["nodeLabels"] = ig.Spec.NodeLabels spec["taints"] = ig.Spec.Taints + spec["suspendProcesses"] = ig.Spec.SuspendProcesses hooks, err := b.getRelevantHooks(ig.Spec.Hooks, ig.Spec.Role) if err != nil { diff --git a/pkg/model/bootstrapscript_test.go b/pkg/model/bootstrapscript_test.go index 027a70913d..cee1ebc08d 100644 --- a/pkg/model/bootstrapscript_test.go +++ b/pkg/model/bootstrapscript_test.go @@ -261,6 +261,9 @@ func makeTestInstanceGroup(role kops.InstanceGroupRole, hookSpecRoles []kops.Ins "key1=value1:NoSchedule", "key2=value2:NoExecute", }, + SuspendProcesses: []string{ + "AZRebalance", + }, Hooks: []kops.HookSpec{ { Name: "disable-update-engine.service", diff --git a/pkg/model/tests/data/bootstrapscript_0.txt b/pkg/model/tests/data/bootstrapscript_0.txt index fcaf1cb6e8..be22580622 100644 --- a/pkg/model/tests/data/bootstrapscript_0.txt +++ b/pkg/model/tests/data/bootstrapscript_0.txt @@ -195,6 +195,8 @@ kubelet: nodeLabels: label2: value2 labelname: labelvalue +suspendProcesses: +- AZRebalance taints: - key1=value1:NoSchedule - key2=value2:NoExecute diff --git a/pkg/model/tests/data/bootstrapscript_1.txt b/pkg/model/tests/data/bootstrapscript_1.txt index 0ce92cf8d1..82d1ad1479 100644 --- a/pkg/model/tests/data/bootstrapscript_1.txt +++ b/pkg/model/tests/data/bootstrapscript_1.txt @@ -212,6 +212,8 @@ kubelet: nodeLabels: label2: value2 labelname: labelvalue +suspendProcesses: +- AZRebalance taints: - key1=value1:NoSchedule - key2=value2:NoExecute diff --git a/pkg/model/tests/data/bootstrapscript_2.txt b/pkg/model/tests/data/bootstrapscript_2.txt index 0ce92cf8d1..82d1ad1479 100644 --- a/pkg/model/tests/data/bootstrapscript_2.txt +++ b/pkg/model/tests/data/bootstrapscript_2.txt @@ -212,6 +212,8 @@ kubelet: nodeLabels: label2: value2 labelname: labelvalue +suspendProcesses: +- AZRebalance taints: - key1=value1:NoSchedule - key2=value2:NoExecute diff --git a/pkg/model/tests/data/bootstrapscript_3.txt b/pkg/model/tests/data/bootstrapscript_3.txt index 6a6f844046..f31c2e17f1 100644 --- a/pkg/model/tests/data/bootstrapscript_3.txt +++ b/pkg/model/tests/data/bootstrapscript_3.txt @@ -180,6 +180,8 @@ kubelet: nodeLabels: label2: value2 labelname: labelvalue +suspendProcesses: +- AZRebalance taints: - key1=value1:NoSchedule - key2=value2:NoExecute diff --git a/pkg/model/tests/data/bootstrapscript_4.txt b/pkg/model/tests/data/bootstrapscript_4.txt index 2a29d29de3..4169dcf904 100644 --- a/pkg/model/tests/data/bootstrapscript_4.txt +++ b/pkg/model/tests/data/bootstrapscript_4.txt @@ -197,6 +197,8 @@ kubelet: nodeLabels: label2: value2 labelname: labelvalue +suspendProcesses: +- AZRebalance taints: - key1=value1:NoSchedule - key2=value2:NoExecute diff --git a/pkg/model/tests/data/bootstrapscript_5.txt b/pkg/model/tests/data/bootstrapscript_5.txt index 2a29d29de3..4169dcf904 100644 --- a/pkg/model/tests/data/bootstrapscript_5.txt +++ b/pkg/model/tests/data/bootstrapscript_5.txt @@ -197,6 +197,8 @@ kubelet: nodeLabels: label2: value2 labelname: labelvalue +suspendProcesses: +- AZRebalance taints: - key1=value1:NoSchedule - key2=value2:NoExecute diff --git a/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml index d21fda237f..6307c68615 100644 --- a/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml @@ -255,6 +255,7 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersadditionaluserda cat > ig_spec.yaml << '__EOF_IG_SPEC' kubelet: null nodeLabels: null + suspendProcesses: null taints: null __EOF_IG_SPEC @@ -483,6 +484,7 @@ Resources.AWSAutoScalingLaunchConfigurationnodesadditionaluserdataexamplecom.Pro cat > ig_spec.yaml << '__EOF_IG_SPEC' kubelet: null nodeLabels: null + suspendProcesses: null taints: null __EOF_IG_SPEC diff --git a/tests/integration/update_cluster/complex/in-v1alpha2.yaml b/tests/integration/update_cluster/complex/in-v1alpha2.yaml index 0b8d201667..ab6de7eb0e 100644 --- a/tests/integration/update_cluster/complex/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/complex/in-v1alpha2.yaml @@ -64,6 +64,8 @@ spec: - sg-exampleid3 - sg-exampleid4 associatePublicIp: true + suspendProcesses: + - AZRebalance image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 machineType: t2.medium maxSize: 2 diff --git a/tests/integration/update_cluster/complex/kubernetes.tf b/tests/integration/update_cluster/complex/kubernetes.tf index 96aee9ddb8..d53dcdacc8 100644 --- a/tests/integration/update_cluster/complex/kubernetes.tf +++ b/tests/integration/update_cluster/complex/kubernetes.tf @@ -127,6 +127,7 @@ resource "aws_autoscaling_group" "nodes-complex-example-com" { metrics_granularity = "1Minute" enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + suspended_processes = ["AZRebalance"] } resource "aws_ebs_volume" "us-test-1a-etcd-events-complex-example-com" { diff --git a/tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml index b78662ee32..5858de70b9 100644 --- a/tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml @@ -246,6 +246,7 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersminimalexampleco cat > ig_spec.yaml << '__EOF_IG_SPEC' kubelet: null nodeLabels: null + suspendProcesses: null taints: null __EOF_IG_SPEC @@ -453,6 +454,7 @@ Resources.AWSAutoScalingLaunchConfigurationnodesminimalexamplecom.Properties.Use cat > ig_spec.yaml << '__EOF_IG_SPEC' kubelet: null nodeLabels: null + suspendProcesses: null taints: null __EOF_IG_SPEC diff --git a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go index b0338c933c..aaa082a60d 100644 --- a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go +++ b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go @@ -48,6 +48,8 @@ type AutoscalingGroup struct { Metrics []string LaunchConfiguration *LaunchConfiguration + + SuspendProcesses []string } var _ fi.CompareWithID = &AutoscalingGroup{} @@ -304,6 +306,21 @@ func (_ *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos } } + if e.SuspendProcesses != nil { + processQuery := &autoscaling.ScalingProcessQuery{} + processQuery.AutoScalingGroupName = e.Name + processQuery.ScalingProcesses = []*string{} + + for _, p := range e.SuspendProcesses { + processQuery.ScalingProcesses = append(processQuery.ScalingProcesses, &p) + } + + _, err := t.Cloud.Autoscaling().SuspendProcesses(processQuery) + if err != nil { + return fmt.Errorf("error suspending processes: %v", err) + } + } + // TODO: Use PropagateAtLaunch = false for tagging? return nil // We have @@ -341,9 +358,11 @@ type terraformAutoscalingGroup struct { Tags []*terraformASGTag `json:"tag,omitempty"` MetricsGranularity *string `json:"metrics_granularity,omitempty"` EnabledMetrics []*string `json:"enabled_metrics,omitempty"` + SuspendedProcesses []*string `json:"suspended_processes, omitempty"` } func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *AutoscalingGroup) error { + tf := &terraformAutoscalingGroup{ Name: e.Name, MinSize: e.MinSize, @@ -405,6 +424,14 @@ func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, c } } + var processes []*string + if e.SuspendProcesses != nil { + for _, p := range e.SuspendProcesses { + processes = append(processes, fi.String(p)) + } + } + tf.SuspendedProcesses = processes + return t.RenderResource("aws_autoscaling_group", *e.Name, tf) }