Merge pull request #4677 from usabilla/external-load-balancers

Add the ability to specify external loadbalancers for instancegroups

cmd/kops/integration_test.go

@@ -169,6 +169,11 @@ func TestPhaseNetwork(t *testing.T) {
 	runTestPhase(t, "lifecyclephases.example.com", "lifecycle_phases", "v1alpha2", true, 1, cloudup.PhaseNetwork)
 }
 
+func TestExternalLoadBalancer(t *testing.T) {
+	runTestAWS(t, "externallb.example.com", "externallb", "v1alpha2", false, 1, true)
+	runTestCloudformation(t, "externallb.example.com", "externallb", "v1alpha2", false, nil)
+}
+
 // TestPhaseIAM tests the output of tf for the iam phase
 func TestPhaseIAM(t *testing.T) {
 	t.Skip("unable to test w/o allowing failed validation")

docs/development/api_updates.md

@@ -10,16 +10,7 @@ that directory directly (the unversioned API) and in the versioned subdirectorie
 
 ## Updating the generated API code
 
-You will need a few tools from kubernetes (these will likely be moved to a shared repo soon):
+To generate the API code simply run `make apimachinery && make` to update the generated API machinery code (conversion functions).  Note
-
-```
-go get k8s.io/kubernetes
-cd ${GOPATH}/src/k8s.io/kubernetes
-git checkout master
-git pull
-```
-
-Then you can run `make apimachinery && make` to update the generated API machinery code (conversion functions).  Note
 that `make apimachinery` (currently) only updates the autogenerated code; it does not trigger a rebuild, hence the
 need for `&& make`.
 

docs/instance_groups.md

@@ -318,6 +318,37 @@ spec:
   - AZRebalance
 ```
 
+## Attaching existing Load Balancers to Instance Groups
+
+Instance groups can be linked to up to 10 load balancers. When attached, any instance launched will
+automatically register itself to the load balancer. For example, if you can create an instance group
+dedicated to running an ingress controller exposed on a
+[NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport), you can
+manually create a load balancer and link it to the instance group. Traffic to the load balancer will now
+automatically go to one of the nodes.
+
+You can specify either `loadBalancerName` to link the instance group to an AWS Classic ELB or you can
+specify `targetGroupARN` to link the instance group to a target group, which are used by Application
+load balancers and Network load balancers.
+
+```
+# Example ingress nodes
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  labels:
+    kops.k8s.io/cluster: k8s.dev.local
+  name: ingress
+spec:
+  machineType: m4.large
+  maxSize: 2
+  minSize: 2
+  role: Node
+  externalLoadBalancers:
+  - targetGroupARN: arn:aws:elasticloadbalancing:eu-west-1:123456789012:targetgroup/my-ingress-target-group/0123456789abcdef
+  - loadBalancerName: my-elb-classic-load-balancer
+```
+
 ## Enabling Detailed-Monitoring on AWS instances
 
 Detailed-Monitoring will cause the monitoring data to be available every 1 minute instead of every 5 minutes. [Enabling Detailed Monitoring](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html). In production environments you may want to consider to enable detailed monitoring for quicker troubleshooting.

pkg/apis/kops/instancegroup.go

@@ -116,6 +116,8 @@ type InstanceGroupSpec struct {
 	AdditionalUserData []UserData `json:"additionalUserData,omitempty"`
 	// SuspendProcesses disables the listed Scaling Policies
 	SuspendProcesses []string `json:"suspendProcesses,omitempty"`
+	// ExternalLoadBalancers define loadbalancers that should be attached to the instancegroup
+	ExternalLoadBalancers []LoadBalancer `json:"externalLoadBalancers,omitempty"`
 	// DetailedInstanceMonitoring defines if detailed-monitoring is enabled (AWS only)
 	DetailedInstanceMonitoring *bool `json:"detailedInstanceMonitoring,omitempty"`
 	// IAMProfileSpec defines the identity of the cloud group iam profile (AWS only).
@@ -206,3 +208,11 @@ func (g *InstanceGroup) AddInstanceGroupNodeLabel() {
 		g.Spec.NodeLabels[NodeLabelInstanceGroup] = g.Name
 	}
 }
+
+// LoadBalancers defines a load balancer
+type LoadBalancer struct {
+	// LoadBalancerName to associate with this instance group (AWS ELB)
+	LoadBalancerName *string `json:"loadBalancerName,omitempty"`
+	// TargetGroupARN to associate with this instance group (AWS ALB/NLB)
+	TargetGroupARN *string `json:"targetGroupArn,omitempty"`
+}

pkg/apis/kops/v1alpha1/instancegroup.go

@@ -96,6 +96,8 @@ type InstanceGroupSpec struct {
 	Zones []string `json:"zones,omitempty"`
 	// SuspendProcesses disables the listed Scaling Policies
 	SuspendProcesses []string `json:"suspendProcesses,omitempty"`
+	// ExternalLoadBalancers define loadbalancers that should be attached to the instancegroup
+	ExternalLoadBalancers []LoadBalancer `json:"externalLoadBalancers,omitempty"`
 	// DetailedInstanceMonitoring defines if detailed-monitoring is enabled (AWS only)
 	DetailedInstanceMonitoring *bool `json:"detailedInstanceMonitoring,omitempty"`
 	// IAMProfileSpec defines the identity of the cloud group iam profile (AWS only).
@@ -119,3 +121,11 @@ type UserData struct {
 	// Content is the user-data content
 	Content string `json:"content,omitempty"`
 }
+
+// LoadBalancers defines a load balancer
+type LoadBalancer struct {
+	// LoadBalancerName to associate with this instance group (AWS ELB)
+	LoadBalancerName *string `json:"loadBalancerName,omitempty"`
+	// TargetGroupARN to associate with this instance group (AWS ALB/NLB)
+	TargetGroupARN *string `json:"targetGroupArn,omitempty"`
+}

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -132,6 +132,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_KuberouterNetworkingSpec_To_v1alpha1_KuberouterNetworkingSpec,
 		Convert_v1alpha1_LeaderElectionConfiguration_To_kops_LeaderElectionConfiguration,
 		Convert_kops_LeaderElectionConfiguration_To_v1alpha1_LeaderElectionConfiguration,
+		Convert_v1alpha1_LoadBalancer_To_kops_LoadBalancer,
+		Convert_kops_LoadBalancer_To_v1alpha1_LoadBalancer,
 		Convert_v1alpha1_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec,
 		Convert_kops_LoadBalancerAccessSpec_To_v1alpha1_LoadBalancerAccessSpec,
 		Convert_v1alpha1_NetworkingSpec_To_kops_NetworkingSpec,
@@ -2001,6 +2003,17 @@ func autoConvert_v1alpha1_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 	}
 	out.Zones = in.Zones
 	out.SuspendProcesses = in.SuspendProcesses
+	if in.ExternalLoadBalancers != nil {
+		in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
+		*out = make([]kops.LoadBalancer, len(*in))
+		for i := range *in {
+			if err := Convert_v1alpha1_LoadBalancer_To_kops_LoadBalancer(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.ExternalLoadBalancers = nil
+	}
 	out.DetailedInstanceMonitoring = in.DetailedInstanceMonitoring
 	if in.IAM != nil {
 		in, out := &in.IAM, &out.IAM
@@ -2076,6 +2089,17 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha1_InstanceGroupSpec(in *kops.I
 		out.AdditionalUserData = nil
 	}
 	out.SuspendProcesses = in.SuspendProcesses
+	if in.ExternalLoadBalancers != nil {
+		in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
+		*out = make([]LoadBalancer, len(*in))
+		for i := range *in {
+			if err := Convert_kops_LoadBalancer_To_v1alpha1_LoadBalancer(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.ExternalLoadBalancers = nil
+	}
 	out.DetailedInstanceMonitoring = in.DetailedInstanceMonitoring
 	if in.IAM != nil {
 		in, out := &in.IAM, &out.IAM
@@ -2683,6 +2707,28 @@ func Convert_kops_LeaderElectionConfiguration_To_v1alpha1_LeaderElectionConfigur
 	return autoConvert_kops_LeaderElectionConfiguration_To_v1alpha1_LeaderElectionConfiguration(in, out, s)
 }
 
+func autoConvert_v1alpha1_LoadBalancer_To_kops_LoadBalancer(in *LoadBalancer, out *kops.LoadBalancer, s conversion.Scope) error {
+	out.LoadBalancerName = in.LoadBalancerName
+	out.TargetGroupARN = in.TargetGroupARN
+	return nil
+}
+
+// Convert_v1alpha1_LoadBalancer_To_kops_LoadBalancer is an autogenerated conversion function.
+func Convert_v1alpha1_LoadBalancer_To_kops_LoadBalancer(in *LoadBalancer, out *kops.LoadBalancer, s conversion.Scope) error {
+	return autoConvert_v1alpha1_LoadBalancer_To_kops_LoadBalancer(in, out, s)
+}
+
+func autoConvert_kops_LoadBalancer_To_v1alpha1_LoadBalancer(in *kops.LoadBalancer, out *LoadBalancer, s conversion.Scope) error {
+	out.LoadBalancerName = in.LoadBalancerName
+	out.TargetGroupARN = in.TargetGroupARN
+	return nil
+}
+
+// Convert_kops_LoadBalancer_To_v1alpha1_LoadBalancer is an autogenerated conversion function.
+func Convert_kops_LoadBalancer_To_v1alpha1_LoadBalancer(in *kops.LoadBalancer, out *LoadBalancer, s conversion.Scope) error {
+	return autoConvert_kops_LoadBalancer_To_v1alpha1_LoadBalancer(in, out, s)
+}
+
 func autoConvert_v1alpha1_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in *LoadBalancerAccessSpec, out *kops.LoadBalancerAccessSpec, s conversion.Scope) error {
 	out.Type = kops.LoadBalancerType(in.Type)
 	out.IdleTimeoutSeconds = in.IdleTimeoutSeconds

pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go

@@ -1765,6 +1765,13 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.ExternalLoadBalancers != nil {
+		in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
+		*out = make([]LoadBalancer, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.DetailedInstanceMonitoring != nil {
 		in, out := &in.DetailedInstanceMonitoring, &out.DetailedInstanceMonitoring
 		if *in == nil {
@@ -2796,6 +2803,40 @@ func (in *LeaderElectionConfiguration) DeepCopy() *LeaderElectionConfiguration {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) {
+	*out = *in
+	if in.LoadBalancerName != nil {
+		in, out := &in.LoadBalancerName, &out.LoadBalancerName
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(string)
+			**out = **in
+		}
+	}
+	if in.TargetGroupARN != nil {
+		in, out := &in.TargetGroupARN, &out.TargetGroupARN
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(string)
+			**out = **in
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer.
+func (in *LoadBalancer) DeepCopy() *LoadBalancer {
+	if in == nil {
+		return nil
+	}
+	out := new(LoadBalancer)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) {
 	*out = *in

pkg/apis/kops/v1alpha2/instancegroup.go

@@ -55,7 +55,7 @@ var AllInstanceGroupRoles = []InstanceGroupRole{
 	InstanceGroupRoleBastion,
 }
 
-// InstanceGroupSpec is the specification for a instanceGroup
+// InstanceGroupSpec is the specification for an instanceGroup
 type InstanceGroupSpec struct {
 	// Type determines the role of instances in this group: masters or nodes
 	Role InstanceGroupRole `json:"role,omitempty"`
@@ -105,6 +105,8 @@ type InstanceGroupSpec struct {
 	AdditionalUserData []UserData `json:"additionalUserData,omitempty"`
 	// SuspendProcesses disables the listed Scaling Policies
 	SuspendProcesses []string `json:"suspendProcesses,omitempty"`
+	// ExternalLoadBalancers define loadbalancers that should be attached to the instancegroup
+	ExternalLoadBalancers []LoadBalancer `json:"externalLoadBalancers,omitempty"`
 	// DetailedInstanceMonitoring defines if detailed-monitoring is enabled (AWS only)
 	DetailedInstanceMonitoring *bool `json:"detailedInstanceMonitoring,omitempty"`
 	// IAMProfileSpec defines the identity of the cloud group iam profile (AWS only).
@@ -128,3 +130,11 @@ type IAMProfileSpec struct {
 	// for the iam instance profile
 	Profile *string `json:"profile,omitempty"`
 }
+
+// LoadBalancer defines a load balancer
+type LoadBalancer struct {
+	// LoadBalancerName to associate with this instance group (AWS ELB)
+	LoadBalancerName *string `json:"loadBalancerName,omitempty"`
+	// TargetGroupARN to associate with this instance group (AWS ALB/NLB)
+	TargetGroupARN *string `json:"targetGroupArn,omitempty"`
+}

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -144,6 +144,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_KuberouterNetworkingSpec_To_v1alpha2_KuberouterNetworkingSpec,
 		Convert_v1alpha2_LeaderElectionConfiguration_To_kops_LeaderElectionConfiguration,
 		Convert_kops_LeaderElectionConfiguration_To_v1alpha2_LeaderElectionConfiguration,
+		Convert_v1alpha2_LoadBalancer_To_kops_LoadBalancer,
+		Convert_kops_LoadBalancer_To_v1alpha2_LoadBalancer,
 		Convert_v1alpha2_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec,
 		Convert_kops_LoadBalancerAccessSpec_To_v1alpha2_LoadBalancerAccessSpec,
 		Convert_v1alpha2_NetworkingSpec_To_kops_NetworkingSpec,
@@ -2113,6 +2115,17 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 		out.AdditionalUserData = nil
 	}
 	out.SuspendProcesses = in.SuspendProcesses
+	if in.ExternalLoadBalancers != nil {
+		in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
+		*out = make([]kops.LoadBalancer, len(*in))
+		for i := range *in {
+			if err := Convert_v1alpha2_LoadBalancer_To_kops_LoadBalancer(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.ExternalLoadBalancers = nil
+	}
 	out.DetailedInstanceMonitoring = in.DetailedInstanceMonitoring
 	if in.IAM != nil {
 		in, out := &in.IAM, &out.IAM
@@ -2193,6 +2206,17 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
 		out.AdditionalUserData = nil
 	}
 	out.SuspendProcesses = in.SuspendProcesses
+	if in.ExternalLoadBalancers != nil {
+		in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
+		*out = make([]LoadBalancer, len(*in))
+		for i := range *in {
+			if err := Convert_kops_LoadBalancer_To_v1alpha2_LoadBalancer(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.ExternalLoadBalancers = nil
+	}
 	out.DetailedInstanceMonitoring = in.DetailedInstanceMonitoring
 	if in.IAM != nil {
 		in, out := &in.IAM, &out.IAM
@@ -2947,6 +2971,28 @@ func Convert_kops_LeaderElectionConfiguration_To_v1alpha2_LeaderElectionConfigur
 	return autoConvert_kops_LeaderElectionConfiguration_To_v1alpha2_LeaderElectionConfiguration(in, out, s)
 }
 
+func autoConvert_v1alpha2_LoadBalancer_To_kops_LoadBalancer(in *LoadBalancer, out *kops.LoadBalancer, s conversion.Scope) error {
+	out.LoadBalancerName = in.LoadBalancerName
+	out.TargetGroupARN = in.TargetGroupARN
+	return nil
+}
+
+// Convert_v1alpha2_LoadBalancer_To_kops_LoadBalancer is an autogenerated conversion function.
+func Convert_v1alpha2_LoadBalancer_To_kops_LoadBalancer(in *LoadBalancer, out *kops.LoadBalancer, s conversion.Scope) error {
+	return autoConvert_v1alpha2_LoadBalancer_To_kops_LoadBalancer(in, out, s)
+}
+
+func autoConvert_kops_LoadBalancer_To_v1alpha2_LoadBalancer(in *kops.LoadBalancer, out *LoadBalancer, s conversion.Scope) error {
+	out.LoadBalancerName = in.LoadBalancerName
+	out.TargetGroupARN = in.TargetGroupARN
+	return nil
+}
+
+// Convert_kops_LoadBalancer_To_v1alpha2_LoadBalancer is an autogenerated conversion function.
+func Convert_kops_LoadBalancer_To_v1alpha2_LoadBalancer(in *kops.LoadBalancer, out *LoadBalancer, s conversion.Scope) error {
+	return autoConvert_kops_LoadBalancer_To_v1alpha2_LoadBalancer(in, out, s)
+}
+
 func autoConvert_v1alpha2_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in *LoadBalancerAccessSpec, out *kops.LoadBalancerAccessSpec, s conversion.Scope) error {
 	out.Type = kops.LoadBalancerType(in.Type)
 	out.IdleTimeoutSeconds = in.IdleTimeoutSeconds

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -1737,6 +1737,13 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.ExternalLoadBalancers != nil {
+		in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
+		*out = make([]LoadBalancer, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.DetailedInstanceMonitoring != nil {
 		in, out := &in.DetailedInstanceMonitoring, &out.DetailedInstanceMonitoring
 		if *in == nil {
@@ -2877,6 +2884,40 @@ func (in *LeaderElectionConfiguration) DeepCopy() *LeaderElectionConfiguration {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) {
+	*out = *in
+	if in.LoadBalancerName != nil {
+		in, out := &in.LoadBalancerName, &out.LoadBalancerName
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(string)
+			**out = **in
+		}
+	}
+	if in.TargetGroupARN != nil {
+		in, out := &in.TargetGroupARN, &out.TargetGroupARN
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(string)
+			**out = **in
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer.
+func (in *LoadBalancer) DeepCopy() *LoadBalancer {
+	if in == nil {
+		return nil
+	}
+	out := new(LoadBalancer)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) {
 	*out = *in

pkg/apis/kops/zz_generated.deepcopy.go

@@ -1918,6 +1918,13 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.ExternalLoadBalancers != nil {
+		in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
+		*out = make([]LoadBalancer, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.DetailedInstanceMonitoring != nil {
 		in, out := &in.DetailedInstanceMonitoring, &out.DetailedInstanceMonitoring
 		if *in == nil {
@@ -3090,6 +3097,40 @@ func (in *LeaderElectionConfiguration) DeepCopy() *LeaderElectionConfiguration {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) {
+	*out = *in
+	if in.LoadBalancerName != nil {
+		in, out := &in.LoadBalancerName, &out.LoadBalancerName
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(string)
+			**out = **in
+		}
+	}
+	if in.TargetGroupARN != nil {
+		in, out := &in.TargetGroupARN, &out.TargetGroupARN
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(string)
+			**out = **in
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer.
+func (in *LoadBalancer) DeepCopy() *LoadBalancer {
+	if in == nil {
+		return nil
+	}
+	out := new(LoadBalancer)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) {
 	*out = *in

pkg/model/awsmodel/autoscalinggroup.go

@@ -238,6 +238,33 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 
 			c.AddTask(t)
 		}
+
+		// External Load Balancer/TargetGroup Attachments
+		{
+			for _, lb := range ig.Spec.ExternalLoadBalancers {
+				if lb.LoadBalancerName != nil {
+					t := &awstasks.ExternalLoadBalancerAttachment{
+						Name:             s("extlb-" + *lb.LoadBalancerName + "-" + ig.Name),
+						Lifecycle:        b.Lifecycle,
+						LoadBalancerName: *lb.LoadBalancerName,
+						AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
+					}
+
+					c.AddTask(t)
+				}
+
+				if lb.TargetGroupARN != nil {
+					t := &awstasks.ExternalTargetGroupAttachment{
+						Name:             s("exttg-" + *lb.TargetGroupARN + "-" + ig.Name),
+						Lifecycle:        b.Lifecycle,
+						TargetGroupARN:   *lb.TargetGroupARN,
+						AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
+					}
+
+					c.AddTask(t)
+				}
+			}
+		}
 	}
 
 	return nil

tests/integration/update_cluster/externallb/cloudformation.json

@@ -0,0 +1,852 @@
+{
+  "Resources": {
+    "AWSAutoScalingAutoScalingGroupmasterustest1amastersexternallbexamplecom": {
+      "Type": "AWS::AutoScaling::AutoScalingGroup",
+      "Properties": {
+        "AutoScalingGroupName": "master-us-test-1a.masters.externallb.example.com",
+        "LaunchConfigurationName": {
+          "Ref": "AWSAutoScalingLaunchConfigurationmasterustest1amastersexternallbexamplecom"
+        },
+        "MaxSize": 1,
+        "MinSize": 1,
+        "VPCZoneIdentifier": [
+          {
+            "Ref": "AWSEC2Subnetustest1aexternallbexamplecom"
+          }
+        ],
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com",
+            "PropagateAtLaunch": true
+          },
+          {
+            "Key": "Name",
+            "Value": "master-us-test-1a.masters.externallb.example.com",
+            "PropagateAtLaunch": true
+          },
+          {
+            "Key": "k8s.io/role/master",
+            "Value": "1",
+            "PropagateAtLaunch": true
+          }
+        ],
+        "MetricsCollection": [
+          {
+            "Granularity": "1Minute",
+            "Metrics": [
+              "GroupDesiredCapacity",
+              "GroupInServiceInstances",
+              "GroupMaxSize",
+              "GroupMinSize",
+              "GroupPendingInstances",
+              "GroupStandbyInstances",
+              "GroupTerminatingInstances",
+              "GroupTotalInstances"
+            ]
+          }
+        ],
+        "LoadBalancerNames": [
+          "my-other-elb"
+        ],
+        "TargetGroupARNs": [
+          "aws:arn:elasticloadbalancing:us-test-1a:123456789012:targetgroup/my-tg/0123456789abcdef"
+        ]
+      }
+    },
+    "AWSAutoScalingAutoScalingGroupnodesexternallbexamplecom": {
+      "Type": "AWS::AutoScaling::AutoScalingGroup",
+      "Properties": {
+        "AutoScalingGroupName": "nodes.externallb.example.com",
+        "LaunchConfigurationName": {
+          "Ref": "AWSAutoScalingLaunchConfigurationnodesexternallbexamplecom"
+        },
+        "MaxSize": 2,
+        "MinSize": 2,
+        "VPCZoneIdentifier": [
+          {
+            "Ref": "AWSEC2Subnetustest1aexternallbexamplecom"
+          }
+        ],
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com",
+            "PropagateAtLaunch": true
+          },
+          {
+            "Key": "Name",
+            "Value": "nodes.externallb.example.com",
+            "PropagateAtLaunch": true
+          },
+          {
+            "Key": "k8s.io/role/node",
+            "Value": "1",
+            "PropagateAtLaunch": true
+          }
+        ],
+        "MetricsCollection": [
+          {
+            "Granularity": "1Minute",
+            "Metrics": [
+              "GroupDesiredCapacity",
+              "GroupInServiceInstances",
+              "GroupMaxSize",
+              "GroupMinSize",
+              "GroupPendingInstances",
+              "GroupStandbyInstances",
+              "GroupTerminatingInstances",
+              "GroupTotalInstances"
+            ]
+          }
+        ],
+        "LoadBalancerNames": [
+          "my-elb"
+        ]
+      }
+    },
+    "AWSAutoScalingLaunchConfigurationmasterustest1amastersexternallbexamplecom": {
+      "Type": "AWS::AutoScaling::LaunchConfiguration",
+      "Properties": {
+        "AssociatePublicIpAddress": true,
+        "BlockDeviceMappings": [
+          {
+            "DeviceName": "/dev/xvda",
+            "Ebs": {
+              "VolumeType": "gp2",
+              "VolumeSize": 64,
+              "DeleteOnTermination": true
+            }
+          },
+          {
+            "DeviceName": "/dev/sdc",
+            "VirtualName": "ephemeral0"
+          }
+        ],
+        "IamInstanceProfile": {
+          "Ref": "AWSIAMInstanceProfilemastersexternallbexamplecom"
+        },
+        "ImageId": "ami-12345678",
+        "InstanceType": "m3.medium",
+        "KeyName": "kubernetes.externallb.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
+        "SecurityGroups": [
+          {
+            "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+          }
+        ],
+        "UserData": "extracted",
+        "InstanceMonitoring": false
+      }
+    },
+    "AWSAutoScalingLaunchConfigurationnodesexternallbexamplecom": {
+      "Type": "AWS::AutoScaling::LaunchConfiguration",
+      "Properties": {
+        "AssociatePublicIpAddress": true,
+        "BlockDeviceMappings": [
+          {
+            "DeviceName": "/dev/xvda",
+            "Ebs": {
+              "VolumeType": "gp2",
+              "VolumeSize": 128,
+              "DeleteOnTermination": true
+            }
+          }
+        ],
+        "IamInstanceProfile": {
+          "Ref": "AWSIAMInstanceProfilenodesexternallbexamplecom"
+        },
+        "ImageId": "ami-12345678",
+        "InstanceType": "t2.medium",
+        "KeyName": "kubernetes.externallb.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
+        "SecurityGroups": [
+          {
+            "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+          }
+        ],
+        "UserData": "extracted",
+        "InstanceMonitoring": false
+      }
+    },
+    "AWSEC2DHCPOptionsexternallbexamplecom": {
+      "Type": "AWS::EC2::DHCPOptions",
+      "Properties": {
+        "DomainName": "us-test-1.compute.internal",
+        "DomainNameServers": [
+          "AmazonProvidedDNS"
+        ],
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "Name",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "kubernetes.io/cluster/externallb.example.com",
+            "Value": "owned"
+          }
+        ]
+      }
+    },
+    "AWSEC2InternetGatewayexternallbexamplecom": {
+      "Type": "AWS::EC2::InternetGateway",
+      "Properties": {
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "Name",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "kubernetes.io/cluster/externallb.example.com",
+            "Value": "owned"
+          }
+        ]
+      }
+    },
+    "AWSEC2Route00000": {
+      "Type": "AWS::EC2::Route",
+      "Properties": {
+        "RouteTableId": {
+          "Ref": "AWSEC2RouteTableexternallbexamplecom"
+        },
+        "DestinationCidrBlock": "0.0.0.0/0",
+        "GatewayId": {
+          "Ref": "AWSEC2InternetGatewayexternallbexamplecom"
+        }
+      }
+    },
+    "AWSEC2RouteTableexternallbexamplecom": {
+      "Type": "AWS::EC2::RouteTable",
+      "Properties": {
+        "VpcId": {
+          "Ref": "AWSEC2VPCexternallbexamplecom"
+        },
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "Name",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "kubernetes.io/cluster/externallb.example.com",
+            "Value": "owned"
+          },
+          {
+            "Key": "kubernetes.io/kops/role",
+            "Value": "public"
+          }
+        ]
+      }
+    },
+    "AWSEC2SecurityGroupEgressmasteregress": {
+      "Type": "AWS::EC2::SecurityGroupEgress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "FromPort": 0,
+        "ToPort": 0,
+        "IpProtocol": "-1",
+        "CidrIp": "0.0.0.0/0"
+      }
+    },
+    "AWSEC2SecurityGroupEgressnodeegress": {
+      "Type": "AWS::EC2::SecurityGroupEgress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+        },
+        "FromPort": 0,
+        "ToPort": 0,
+        "IpProtocol": "-1",
+        "CidrIp": "0.0.0.0/0"
+      }
+    },
+    "AWSEC2SecurityGroupIngressallmastertomaster": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "SourceSecurityGroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "FromPort": 0,
+        "ToPort": 0,
+        "IpProtocol": "-1"
+      }
+    },
+    "AWSEC2SecurityGroupIngressallmastertonode": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+        },
+        "SourceSecurityGroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "FromPort": 0,
+        "ToPort": 0,
+        "IpProtocol": "-1"
+      }
+    },
+    "AWSEC2SecurityGroupIngressallnodetonode": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+        },
+        "SourceSecurityGroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+        },
+        "FromPort": 0,
+        "ToPort": 0,
+        "IpProtocol": "-1"
+      }
+    },
+    "AWSEC2SecurityGroupIngresshttpsexternaltomaster00000": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "FromPort": 443,
+        "ToPort": 443,
+        "IpProtocol": "tcp",
+        "CidrIp": "0.0.0.0/0"
+      }
+    },
+    "AWSEC2SecurityGroupIngressnodetomastertcp12379": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "SourceSecurityGroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+        },
+        "FromPort": 1,
+        "ToPort": 2379,
+        "IpProtocol": "tcp"
+      }
+    },
+    "AWSEC2SecurityGroupIngressnodetomastertcp23824000": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "SourceSecurityGroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+        },
+        "FromPort": 2382,
+        "ToPort": 4000,
+        "IpProtocol": "tcp"
+      }
+    },
+    "AWSEC2SecurityGroupIngressnodetomastertcp400365535": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "SourceSecurityGroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+        },
+        "FromPort": 4003,
+        "ToPort": 65535,
+        "IpProtocol": "tcp"
+      }
+    },
+    "AWSEC2SecurityGroupIngressnodetomasterudp165535": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "SourceSecurityGroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+        },
+        "FromPort": 1,
+        "ToPort": 65535,
+        "IpProtocol": "udp"
+      }
+    },
+    "AWSEC2SecurityGroupIngresssshexternaltomaster00000": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
+        },
+        "FromPort": 22,
+        "ToPort": 22,
+        "IpProtocol": "tcp",
+        "CidrIp": "0.0.0.0/0"
+      }
+    },
+    "AWSEC2SecurityGroupIngresssshexternaltonode00000": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
+        },
+        "FromPort": 22,
+        "ToPort": 22,
+        "IpProtocol": "tcp",
+        "CidrIp": "0.0.0.0/0"
+      }
+    },
+    "AWSEC2SecurityGroupmastersexternallbexamplecom": {
+      "Type": "AWS::EC2::SecurityGroup",
+      "Properties": {
+        "VpcId": {
+          "Ref": "AWSEC2VPCexternallbexamplecom"
+        },
+        "GroupDescription": "Security group for masters",
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "Name",
+            "Value": "masters.externallb.example.com"
+          },
+          {
+            "Key": "kubernetes.io/cluster/externallb.example.com",
+            "Value": "owned"
+          }
+        ]
+      }
+    },
+    "AWSEC2SecurityGroupnodesexternallbexamplecom": {
+      "Type": "AWS::EC2::SecurityGroup",
+      "Properties": {
+        "VpcId": {
+          "Ref": "AWSEC2VPCexternallbexamplecom"
+        },
+        "GroupDescription": "Security group for nodes",
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "Name",
+            "Value": "nodes.externallb.example.com"
+          },
+          {
+            "Key": "kubernetes.io/cluster/externallb.example.com",
+            "Value": "owned"
+          }
+        ]
+      }
+    },
+    "AWSEC2SubnetRouteTableAssociationustest1aexternallbexamplecom": {
+      "Type": "AWS::EC2::SubnetRouteTableAssociation",
+      "Properties": {
+        "SubnetId": {
+          "Ref": "AWSEC2Subnetustest1aexternallbexamplecom"
+        },
+        "RouteTableId": {
+          "Ref": "AWSEC2RouteTableexternallbexamplecom"
+        }
+      }
+    },
+    "AWSEC2Subnetustest1aexternallbexamplecom": {
+      "Type": "AWS::EC2::Subnet",
+      "Properties": {
+        "VpcId": {
+          "Ref": "AWSEC2VPCexternallbexamplecom"
+        },
+        "CidrBlock": "172.20.32.0/19",
+        "AvailabilityZone": "us-test-1a",
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "Name",
+            "Value": "us-test-1a.externallb.example.com"
+          },
+          {
+            "Key": "SubnetType",
+            "Value": "Public"
+          },
+          {
+            "Key": "kubernetes.io/cluster/externallb.example.com",
+            "Value": "owned"
+          },
+          {
+            "Key": "kubernetes.io/role/elb",
+            "Value": "1"
+          }
+        ]
+      }
+    },
+    "AWSEC2VPCDHCPOptionsAssociationexternallbexamplecom": {
+      "Type": "AWS::EC2::VPCDHCPOptionsAssociation",
+      "Properties": {
+        "VpcId": {
+          "Ref": "AWSEC2VPCexternallbexamplecom"
+        },
+        "DhcpOptionsId": {
+          "Ref": "AWSEC2DHCPOptionsexternallbexamplecom"
+        }
+      }
+    },
+    "AWSEC2VPCGatewayAttachmentexternallbexamplecom": {
+      "Type": "AWS::EC2::VPCGatewayAttachment",
+      "Properties": {
+        "VpcId": {
+          "Ref": "AWSEC2VPCexternallbexamplecom"
+        },
+        "InternetGatewayId": {
+          "Ref": "AWSEC2InternetGatewayexternallbexamplecom"
+        }
+      }
+    },
+    "AWSEC2VPCexternallbexamplecom": {
+      "Type": "AWS::EC2::VPC",
+      "Properties": {
+        "CidrBlock": "172.20.0.0/16",
+        "EnableDnsHostnames": true,
+        "EnableDnsSupport": true,
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "Name",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "kubernetes.io/cluster/externallb.example.com",
+            "Value": "owned"
+          }
+        ]
+      }
+    },
+    "AWSEC2Volumeustest1aetcdeventsexternallbexamplecom": {
+      "Type": "AWS::EC2::Volume",
+      "Properties": {
+        "AvailabilityZone": "us-test-1a",
+        "Size": 20,
+        "VolumeType": "gp2",
+        "Encrypted": false,
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "Name",
+            "Value": "us-test-1a.etcd-events.externallb.example.com"
+          },
+          {
+            "Key": "k8s.io/etcd/events",
+            "Value": "us-test-1a/us-test-1a"
+          },
+          {
+            "Key": "k8s.io/role/master",
+            "Value": "1"
+          },
+          {
+            "Key": "kubernetes.io/cluster/externallb.example.com",
+            "Value": "owned"
+          }
+        ]
+      }
+    },
+    "AWSEC2Volumeustest1aetcdmainexternallbexamplecom": {
+      "Type": "AWS::EC2::Volume",
+      "Properties": {
+        "AvailabilityZone": "us-test-1a",
+        "Size": 20,
+        "VolumeType": "gp2",
+        "Encrypted": false,
+        "Tags": [
+          {
+            "Key": "KubernetesCluster",
+            "Value": "externallb.example.com"
+          },
+          {
+            "Key": "Name",
+            "Value": "us-test-1a.etcd-main.externallb.example.com"
+          },
+          {
+            "Key": "k8s.io/etcd/main",
+            "Value": "us-test-1a/us-test-1a"
+          },
+          {
+            "Key": "k8s.io/role/master",
+            "Value": "1"
+          },
+          {
+            "Key": "kubernetes.io/cluster/externallb.example.com",
+            "Value": "owned"
+          }
+        ]
+      }
+    },
+    "AWSIAMInstanceProfilemastersexternallbexamplecom": {
+      "Type": "AWS::IAM::InstanceProfile",
+      "Properties": {
+        "Roles": [
+          {
+            "Ref": "AWSIAMRolemastersexternallbexamplecom"
+          }
+        ]
+      }
+    },
+    "AWSIAMInstanceProfilenodesexternallbexamplecom": {
+      "Type": "AWS::IAM::InstanceProfile",
+      "Properties": {
+        "Roles": [
+          {
+            "Ref": "AWSIAMRolenodesexternallbexamplecom"
+          }
+        ]
+      }
+    },
+    "AWSIAMPolicymastersexternallbexamplecom": {
+      "Type": "AWS::IAM::Policy",
+      "Properties": {
+        "PolicyName": "masters.externallb.example.com",
+        "Roles": [
+          {
+            "Ref": "AWSIAMRolemastersexternallbexamplecom"
+          }
+        ],
+        "PolicyDocument": {
+          "Statement": [
+            {
+              "Action": [
+                "ec2:*"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            },
+            {
+              "Action": [
+                "autoscaling:DescribeAutoScalingGroups",
+                "autoscaling:DescribeAutoScalingInstances",
+                "autoscaling:DescribeLaunchConfigurations",
+                "autoscaling:DescribeTags",
+                "autoscaling:GetAsgForInstance",
+                "autoscaling:SetDesiredCapacity",
+                "autoscaling:TerminateInstanceInAutoScalingGroup",
+                "autoscaling:UpdateAutoScalingGroup"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            },
+            {
+              "Action": [
+                "elasticloadbalancing:*"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            },
+            {
+              "Action": [
+                "iam:ListServerCertificates",
+                "iam:GetServerCertificate"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            },
+            {
+              "Action": [
+                "route53:ChangeResourceRecordSets",
+                "route53:ListResourceRecordSets",
+                "route53:GetHostedZone"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+              ]
+            },
+            {
+              "Action": [
+                "route53:GetChange"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "arn:aws:route53:::change/*"
+              ]
+            },
+            {
+              "Action": [
+                "route53:ListHostedZones"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            },
+            {
+              "Action": [
+                "route53:ListHostedZones"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            },
+            {
+              "Action": [
+                "ecr:GetAuthorizationToken",
+                "ecr:BatchCheckLayerAvailability",
+                "ecr:GetDownloadUrlForLayer",
+                "ecr:GetRepositoryPolicy",
+                "ecr:DescribeRepositories",
+                "ecr:ListImages",
+                "ecr:BatchGetImage"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            }
+          ],
+          "Version": "2012-10-17"
+        }
+      }
+    },
+    "AWSIAMPolicynodesexternallbexamplecom": {
+      "Type": "AWS::IAM::Policy",
+      "Properties": {
+        "PolicyName": "nodes.externallb.example.com",
+        "Roles": [
+          {
+            "Ref": "AWSIAMRolenodesexternallbexamplecom"
+          }
+        ],
+        "PolicyDocument": {
+          "Statement": [
+            {
+              "Action": [
+                "ec2:DescribeInstances",
+                "ec2:DescribeRegions"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            },
+            {
+              "Action": [
+                "route53:ChangeResourceRecordSets",
+                "route53:ListResourceRecordSets",
+                "route53:GetHostedZone"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+              ]
+            },
+            {
+              "Action": [
+                "route53:GetChange"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "arn:aws:route53:::change/*"
+              ]
+            },
+            {
+              "Action": [
+                "route53:ListHostedZones"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            },
+            {
+              "Action": [
+                "route53:ListHostedZones"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            },
+            {
+              "Action": [
+                "ecr:GetAuthorizationToken",
+                "ecr:BatchCheckLayerAvailability",
+                "ecr:GetDownloadUrlForLayer",
+                "ecr:GetRepositoryPolicy",
+                "ecr:DescribeRepositories",
+                "ecr:ListImages",
+                "ecr:BatchGetImage"
+              ],
+              "Effect": "Allow",
+              "Resource": [
+                "*"
+              ]
+            }
+          ],
+          "Version": "2012-10-17"
+        }
+      }
+    },
+    "AWSIAMRolemastersexternallbexamplecom": {
+      "Type": "AWS::IAM::Role",
+      "Properties": {
+        "RoleName": "masters.externallb.example.com",
+        "AssumeRolePolicyDocument": {
+          "Statement": [
+            {
+              "Action": "sts:AssumeRole",
+              "Effect": "Allow",
+              "Principal": {
+                "Service": "ec2.amazonaws.com"
+              }
+            }
+          ],
+          "Version": "2012-10-17"
+        }
+      }
+    },
+    "AWSIAMRolenodesexternallbexamplecom": {
+      "Type": "AWS::IAM::Role",
+      "Properties": {
+        "RoleName": "nodes.externallb.example.com",
+        "AssumeRolePolicyDocument": {
+          "Statement": [
+            {
+              "Action": "sts:AssumeRole",
+              "Effect": "Allow",
+              "Principal": {
+                "Service": "ec2.amazonaws.com"
+              }
+            }
+          ],
+          "Version": "2012-10-17"
+        }
+      }
+    }
+  }
+}

tests/integration/update_cluster/externallb/cloudformation.json.extracted.yaml

@@ -0,0 +1,507 @@
+Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersexternallbexamplecom.Properties.UserData: |
+  #!/bin/bash
+  # Copyright 2016 The Kubernetes Authors All rights reserved.
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  NODEUP_URL=https://kubeupv2.s3.amazonaws.com/kops/1.8.1/linux/amd64/nodeup
+  NODEUP_HASH=bb41724c37d15ab7e039e06230e742b9b38d0808
+
+  export AWS_REGION=us-test-1
+
+
+
+
+  function ensure-install-dir() {
+    INSTALL_DIR="/var/cache/kubernetes-install"
+    # On ContainerOS, we install to /var/lib/toolbox install (because of noexec)
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kubernetes-install"
+    fi
+    mkdir -p ${INSTALL_DIR}
+    cd ${INSTALL_DIR}
+  }
+
+  # Retry a download until we get it. Takes a hash and a set of URLs.
+  #
+  # $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
+  # $2+ are the URLs to download.
+  download-or-bust() {
+    local -r hash="$1"
+    shift 1
+
+    urls=( $* )
+    while true; do
+      for url in "${urls[@]}"; do
+        local file="${url##*/}"
+        rm -f "${file}"
+
+        if [[ $(which curl) ]]; then
+          if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
+            echo "== Failed to curl ${url}. Retrying. =="
+            break
+          fi
+        elif [[ $(which wget ) ]]; then
+          if ! wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10 "${url}"; then
+            echo "== Failed to wget ${url}. Retrying. =="
+            break
+          fi
+        else
+          echo "== Could not find curl or wget. Retrying. =="
+          break
+        fi
+
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+
+      echo "All downloads failed; sleeping before retrying"
+      sleep 60
+    done
+  }
+
+  validate-hash() {
+    local -r file="$1"
+    local -r expected="$2"
+    local actual
+
+    actual=$(sha1sum ${file} | awk '{ print $1 }') || true
+    if [[ "${actual}" != "${expected}" ]]; then
+      echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
+      return 1
+    fi
+  }
+
+  function split-commas() {
+    echo $1 | tr "," "\n"
+  }
+
+  function try-download-release() {
+    # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot
+    # optimization.
+
+    local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+    local -r nodeup_filename="${nodeup_urls[0]##*/}"
+    if [[ -n "${NODEUP_HASH:-}" ]]; then
+      local -r nodeup_hash="${NODEUP_HASH}"
+    else
+    # TODO: Remove?
+      echo "Downloading sha1 (not found in env)"
+      download-or-bust "" "${nodeup_urls[@]/%/.sha1}"
+      local -r nodeup_hash=$(cat "${nodeup_filename}.sha1")
+    fi
+
+    echo "Downloading nodeup (${nodeup_urls[@]})"
+    download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}"
+
+    chmod +x nodeup
+  }
+
+  function download-release() {
+    # In case of failure checking integrity of release, retry.
+    until try-download-release; do
+      sleep 15
+      echo "Couldn't download release. Retrying..."
+    done
+
+    echo "Running nodeup"
+    # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+    ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8  )
+  }
+
+  ####################################################################################
+
+  /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+  echo "== nodeup node config starting =="
+  ensure-install-dir
+
+  cat > cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+  cloudConfig: null
+  docker:
+    ipMasq: false
+    ipTables: false
+    logDriver: json-file
+    logLevel: warn
+    logOpt:
+    - max-size=10m
+    - max-file=5
+    storage: overlay,aufs
+    version: 1.13.1
+  encryptionConfig: null
+  etcdClusters:
+    events:
+      image: gcr.io/google_containers/etcd:2.2.1
+      version: 2.2.1
+    main:
+      image: gcr.io/google_containers/etcd:2.2.1
+      version: 2.2.1
+  kubeAPIServer:
+    address: 127.0.0.1
+    admissionControl:
+    - Initializers
+    - NamespaceLifecycle
+    - LimitRanger
+    - ServiceAccount
+    - PersistentVolumeLabel
+    - DefaultStorageClass
+    - DefaultTolerationSeconds
+    - NodeRestriction
+    - ResourceQuota
+    allowPrivileged: true
+    anonymousAuth: false
+    apiServerCount: 1
+    authorizationMode: AlwaysAllow
+    cloudProvider: aws
+    etcdServers:
+    - http://127.0.0.1:4001
+    etcdServersOverrides:
+    - /events#http://127.0.0.1:4002
+    image: gcr.io/google_containers/kube-apiserver:v1.8.0
+    insecurePort: 8080
+    kubeletPreferredAddressTypes:
+    - InternalIP
+    - Hostname
+    - ExternalIP
+    logLevel: 2
+    requestheaderAllowedNames:
+    - aggregator
+    requestheaderExtraHeaderPrefixes:
+    - X-Remote-Extra-
+    requestheaderGroupHeaders:
+    - X-Remote-Group
+    requestheaderUsernameHeaders:
+    - X-Remote-User
+    securePort: 443
+    serviceClusterIPRange: 100.64.0.0/13
+    storageBackend: etcd2
+  kubeControllerManager:
+    allocateNodeCIDRs: true
+    attachDetachReconcileSyncPeriod: 1m0s
+    cloudProvider: aws
+    clusterCIDR: 100.96.0.0/11
+    clusterName: externallb.example.com
+    configureCloudRoutes: true
+    image: gcr.io/google_containers/kube-controller-manager:v1.8.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+    useServiceAccountCredentials: true
+  kubeProxy:
+    clusterCIDR: 100.96.0.0/11
+    cpuRequest: 100m
+    hostnameOverride: '@aws'
+    image: gcr.io/google_containers/kube-proxy:v1.8.0
+    logLevel: 2
+  kubeScheduler:
+    image: gcr.io/google_containers/kube-scheduler:v1.8.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+  kubelet:
+    allowPrivileged: true
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      ExperimentalCriticalPodAnnotation: "true"
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    networkPluginMTU: 9001
+    networkPluginName: kubenet
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podInfraContainerImage: gcr.io/google_containers/pause-amd64:3.0
+    podManifestPath: /etc/kubernetes/manifests
+    requireKubeconfig: true
+  masterKubelet:
+    allowPrivileged: true
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      ExperimentalCriticalPodAnnotation: "true"
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    networkPluginMTU: 9001
+    networkPluginName: kubenet
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podInfraContainerImage: gcr.io/google_containers/pause-amd64:3.0
+    podManifestPath: /etc/kubernetes/manifests
+    registerSchedulable: false
+    requireKubeconfig: true
+
+  __EOF_CLUSTER_SPEC
+
+  cat > ig_spec.yaml << '__EOF_IG_SPEC'
+  kubelet: null
+  nodeLabels: null
+  suspendProcesses: null
+  taints: null
+
+  __EOF_IG_SPEC
+
+  cat > kube_env.yaml << '__EOF_KUBE_ENV'
+  Assets:
+  - 4c7b8aafe652ae107c9131754a2ad4e9641a025b@https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubelet
+  - 006fd43085e6ba2dc6b35b89af4d68cee3f689c9@https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubectl
+  - 1d9788b0f5420e1a219aad2cb8681823fc515e7c@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff.tar.gz
+  - 42b15a0a0a56531750bde3c7b08d0cf27c170c48@https://kubeupv2.s3.amazonaws.com/kops/1.8.1/linux/amd64/utils.tar.gz
+  ClusterName: externallb.example.com
+  ConfigBase: memfs://clusters.example.com/externallb.example.com
+  InstanceGroupName: master-us-test-1a
+  Tags:
+  - _automatic_upgrades
+  - _aws
+  - _kubernetes_master
+  channels:
+  - memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml
+  protokubeImage:
+    hash: 0b1f26208f8f6cc02468368706d0236670fec8a2
+    name: protokube:1.8.1
+    source: https://kubeupv2.s3.amazonaws.com/kops/1.8.1/images/protokube.tar.gz
+
+  __EOF_KUBE_ENV
+
+  download-release
+  echo "== nodeup node config done =="
+Resources.AWSAutoScalingLaunchConfigurationnodesexternallbexamplecom.Properties.UserData: |
+  #!/bin/bash
+  # Copyright 2016 The Kubernetes Authors All rights reserved.
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  NODEUP_URL=https://kubeupv2.s3.amazonaws.com/kops/1.8.1/linux/amd64/nodeup
+  NODEUP_HASH=bb41724c37d15ab7e039e06230e742b9b38d0808
+
+  export AWS_REGION=us-test-1
+
+
+
+
+  function ensure-install-dir() {
+    INSTALL_DIR="/var/cache/kubernetes-install"
+    # On ContainerOS, we install to /var/lib/toolbox install (because of noexec)
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kubernetes-install"
+    fi
+    mkdir -p ${INSTALL_DIR}
+    cd ${INSTALL_DIR}
+  }
+
+  # Retry a download until we get it. Takes a hash and a set of URLs.
+  #
+  # $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
+  # $2+ are the URLs to download.
+  download-or-bust() {
+    local -r hash="$1"
+    shift 1
+
+    urls=( $* )
+    while true; do
+      for url in "${urls[@]}"; do
+        local file="${url##*/}"
+        rm -f "${file}"
+
+        if [[ $(which curl) ]]; then
+          if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
+            echo "== Failed to curl ${url}. Retrying. =="
+            break
+          fi
+        elif [[ $(which wget ) ]]; then
+          if ! wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10 "${url}"; then
+            echo "== Failed to wget ${url}. Retrying. =="
+            break
+          fi
+        else
+          echo "== Could not find curl or wget. Retrying. =="
+          break
+        fi
+
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+
+      echo "All downloads failed; sleeping before retrying"
+      sleep 60
+    done
+  }
+
+  validate-hash() {
+    local -r file="$1"
+    local -r expected="$2"
+    local actual
+
+    actual=$(sha1sum ${file} | awk '{ print $1 }') || true
+    if [[ "${actual}" != "${expected}" ]]; then
+      echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
+      return 1
+    fi
+  }
+
+  function split-commas() {
+    echo $1 | tr "," "\n"
+  }
+
+  function try-download-release() {
+    # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot
+    # optimization.
+
+    local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+    local -r nodeup_filename="${nodeup_urls[0]##*/}"
+    if [[ -n "${NODEUP_HASH:-}" ]]; then
+      local -r nodeup_hash="${NODEUP_HASH}"
+    else
+    # TODO: Remove?
+      echo "Downloading sha1 (not found in env)"
+      download-or-bust "" "${nodeup_urls[@]/%/.sha1}"
+      local -r nodeup_hash=$(cat "${nodeup_filename}.sha1")
+    fi
+
+    echo "Downloading nodeup (${nodeup_urls[@]})"
+    download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}"
+
+    chmod +x nodeup
+  }
+
+  function download-release() {
+    # In case of failure checking integrity of release, retry.
+    until try-download-release; do
+      sleep 15
+      echo "Couldn't download release. Retrying..."
+    done
+
+    echo "Running nodeup"
+    # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+    ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8  )
+  }
+
+  ####################################################################################
+
+  /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+  echo "== nodeup node config starting =="
+  ensure-install-dir
+
+  cat > cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+  cloudConfig: null
+  docker:
+    ipMasq: false
+    ipTables: false
+    logDriver: json-file
+    logLevel: warn
+    logOpt:
+    - max-size=10m
+    - max-file=5
+    storage: overlay,aufs
+    version: 1.13.1
+  kubeProxy:
+    clusterCIDR: 100.96.0.0/11
+    cpuRequest: 100m
+    hostnameOverride: '@aws'
+    image: gcr.io/google_containers/kube-proxy:v1.8.0
+    logLevel: 2
+  kubelet:
+    allowPrivileged: true
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      ExperimentalCriticalPodAnnotation: "true"
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    networkPluginMTU: 9001
+    networkPluginName: kubenet
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podInfraContainerImage: gcr.io/google_containers/pause-amd64:3.0
+    podManifestPath: /etc/kubernetes/manifests
+    requireKubeconfig: true
+
+  __EOF_CLUSTER_SPEC
+
+  cat > ig_spec.yaml << '__EOF_IG_SPEC'
+  kubelet: null
+  nodeLabels: null
+  suspendProcesses: null
+  taints: null
+
+  __EOF_IG_SPEC
+
+  cat > kube_env.yaml << '__EOF_KUBE_ENV'
+  Assets:
+  - 4c7b8aafe652ae107c9131754a2ad4e9641a025b@https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubelet
+  - 006fd43085e6ba2dc6b35b89af4d68cee3f689c9@https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubectl
+  - 1d9788b0f5420e1a219aad2cb8681823fc515e7c@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff.tar.gz
+  - 42b15a0a0a56531750bde3c7b08d0cf27c170c48@https://kubeupv2.s3.amazonaws.com/kops/1.8.1/linux/amd64/utils.tar.gz
+  ClusterName: externallb.example.com
+  ConfigBase: memfs://clusters.example.com/externallb.example.com
+  InstanceGroupName: nodes
+  Tags:
+  - _automatic_upgrades
+  - _aws
+  channels:
+  - memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml
+  protokubeImage:
+    hash: 0b1f26208f8f6cc02468368706d0236670fec8a2
+    name: protokube:1.8.1
+    source: https://kubeupv2.s3.amazonaws.com/kops/1.8.1/images/protokube.tar.gz
+
+  __EOF_KUBE_ENV
+
+  download-release
+  echo "== nodeup node config done =="

tests/integration/update_cluster/externallb/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/externallb/in-v1alpha2.yaml

@@ -0,0 +1,82 @@
+apiVersion: kops/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2018-03-20T16:00:27Z"
+  name: externallb.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/externallb.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: events
+  kubernetesVersion: v1.8.0
+  masterInternalName: api.internal.externallb.example.com
+  masterPublicName: api.externallb.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: nodes
+  labels:
+    kops.k8s.io/cluster: externallb.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: t2.medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test-1a
+  externalLoadBalancers:
+  - loadBalancerName: my-elb
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: externallb.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a
+  externalLoadBalancers:
+  - targetGroupArn: aws:arn:elasticloadbalancing:us-test-1a:123456789012:targetgroup/my-tg/0123456789abcdef
+  - loadBalancerName: my-other-elb
+
+

tests/integration/update_cluster/externallb/kubernetes.tf

@@ -0,0 +1,456 @@
+output "cluster_name" {
+  value = "externallb.example.com"
+}
+
+output "master_security_group_ids" {
+  value = ["${aws_security_group.masters-externallb-example-com.id}"]
+}
+
+output "masters_role_arn" {
+  value = "${aws_iam_role.masters-externallb-example-com.arn}"
+}
+
+output "masters_role_name" {
+  value = "${aws_iam_role.masters-externallb-example-com.name}"
+}
+
+output "node_security_group_ids" {
+  value = ["${aws_security_group.nodes-externallb-example-com.id}"]
+}
+
+output "node_subnet_ids" {
+  value = ["${aws_subnet.us-test-1a-externallb-example-com.id}"]
+}
+
+output "nodes_role_arn" {
+  value = "${aws_iam_role.nodes-externallb-example-com.arn}"
+}
+
+output "nodes_role_name" {
+  value = "${aws_iam_role.nodes-externallb-example-com.name}"
+}
+
+output "region" {
+  value = "us-test-1"
+}
+
+output "route_table_public_id" {
+  value = "${aws_route_table.externallb-example-com.id}"
+}
+
+output "subnet_us-test-1a-public_id" {
+  value = "${aws_subnet.us-test-1a-externallb-example-com.id}"
+}
+
+output "vpc_cidr_block" {
+  value = "${aws_vpc.externallb-example-com.cidr_block}"
+}
+
+output "vpc_id" {
+  value = "${aws_vpc.externallb-example-com.id}"
+}
+
+provider "aws" {
+  region = "us-test-1"
+}
+
+resource "aws_autoscaling_attachment" "extlb-my-elb-nodes" {
+  elb                    = "my-elb"
+  autoscaling_group_name = "${aws_autoscaling_group.nodes-externallb-example-com.id}"
+}
+
+resource "aws_autoscaling_attachment" "extlb-my-other-elb-master-us-test-1a" {
+  elb                    = "my-other-elb"
+  autoscaling_group_name = "${aws_autoscaling_group.master-us-test-1a-masters-externallb-example-com.id}"
+}
+
+resource "aws_autoscaling_attachment" "exttg-aws:my-tg--0123456789abcdef-master-us-test-1a" {
+  alb_target_group_arn   = "aws:arn:elasticloadbalancing:us-test-1a:123456789012:targetgroup/my-tg/0123456789abcdef"
+  autoscaling_group_name = "${aws_autoscaling_group.master-us-test-1a-masters-externallb-example-com.id}"
+}
+
+resource "aws_autoscaling_group" "master-us-test-1a-masters-externallb-example-com" {
+  name                 = "master-us-test-1a.masters.externallb.example.com"
+  launch_configuration = "${aws_launch_configuration.master-us-test-1a-masters-externallb-example-com.id}"
+  max_size             = 1
+  min_size             = 1
+  vpc_zone_identifier  = ["${aws_subnet.us-test-1a-externallb-example-com.id}"]
+
+  tag = {
+    key                 = "KubernetesCluster"
+    value               = "externallb.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "Name"
+    value               = "master-us-test-1a.masters.externallb.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "k8s.io/role/master"
+    value               = "1"
+    propagate_at_launch = true
+  }
+
+  metrics_granularity = "1Minute"
+  enabled_metrics     = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+}
+
+resource "aws_autoscaling_group" "nodes-externallb-example-com" {
+  name                 = "nodes.externallb.example.com"
+  launch_configuration = "${aws_launch_configuration.nodes-externallb-example-com.id}"
+  max_size             = 2
+  min_size             = 2
+  vpc_zone_identifier  = ["${aws_subnet.us-test-1a-externallb-example-com.id}"]
+
+  tag = {
+    key                 = "KubernetesCluster"
+    value               = "externallb.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "Name"
+    value               = "nodes.externallb.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "k8s.io/role/node"
+    value               = "1"
+    propagate_at_launch = true
+  }
+
+  metrics_granularity = "1Minute"
+  enabled_metrics     = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-events-externallb-example-com" {
+  availability_zone = "us-test-1a"
+  size              = 20
+  type              = "gp2"
+  encrypted         = false
+
+  tags = {
+    KubernetesCluster                              = "externallb.example.com"
+    Name                                           = "us-test-1a.etcd-events.externallb.example.com"
+    "k8s.io/etcd/events"                           = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                           = "1"
+    "kubernetes.io/cluster/externallb.example.com" = "owned"
+  }
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-main-externallb-example-com" {
+  availability_zone = "us-test-1a"
+  size              = 20
+  type              = "gp2"
+  encrypted         = false
+
+  tags = {
+    KubernetesCluster                              = "externallb.example.com"
+    Name                                           = "us-test-1a.etcd-main.externallb.example.com"
+    "k8s.io/etcd/main"                             = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                           = "1"
+    "kubernetes.io/cluster/externallb.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_instance_profile" "masters-externallb-example-com" {
+  name = "masters.externallb.example.com"
+  role = "${aws_iam_role.masters-externallb-example-com.name}"
+}
+
+resource "aws_iam_instance_profile" "nodes-externallb-example-com" {
+  name = "nodes.externallb.example.com"
+  role = "${aws_iam_role.nodes-externallb-example-com.name}"
+}
+
+resource "aws_iam_role" "masters-externallb-example-com" {
+  name               = "masters.externallb.example.com"
+  assume_role_policy = "${file("${path.module}/data/aws_iam_role_masters.externallb.example.com_policy")}"
+}
+
+resource "aws_iam_role" "nodes-externallb-example-com" {
+  name               = "nodes.externallb.example.com"
+  assume_role_policy = "${file("${path.module}/data/aws_iam_role_nodes.externallb.example.com_policy")}"
+}
+
+resource "aws_iam_role_policy" "masters-externallb-example-com" {
+  name   = "masters.externallb.example.com"
+  role   = "${aws_iam_role.masters-externallb-example-com.name}"
+  policy = "${file("${path.module}/data/aws_iam_role_policy_masters.externallb.example.com_policy")}"
+}
+
+resource "aws_iam_role_policy" "nodes-externallb-example-com" {
+  name   = "nodes.externallb.example.com"
+  role   = "${aws_iam_role.nodes-externallb-example-com.name}"
+  policy = "${file("${path.module}/data/aws_iam_role_policy_nodes.externallb.example.com_policy")}"
+}
+
+resource "aws_internet_gateway" "externallb-example-com" {
+  vpc_id = "${aws_vpc.externallb-example-com.id}"
+
+  tags = {
+    KubernetesCluster                              = "externallb.example.com"
+    Name                                           = "externallb.example.com"
+    "kubernetes.io/cluster/externallb.example.com" = "owned"
+  }
+}
+
+resource "aws_key_pair" "kubernetes-externallb-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
+  key_name   = "kubernetes.externallb.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
+  public_key = "${file("${path.module}/data/aws_key_pair_kubernetes.externallb.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")}"
+}
+
+resource "aws_launch_configuration" "master-us-test-1a-masters-externallb-example-com" {
+  name_prefix                 = "master-us-test-1a.masters.externallb.example.com-"
+  image_id                    = "ami-12345678"
+  instance_type               = "m3.medium"
+  key_name                    = "${aws_key_pair.kubernetes-externallb-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
+  iam_instance_profile        = "${aws_iam_instance_profile.masters-externallb-example-com.id}"
+  security_groups             = ["${aws_security_group.masters-externallb-example-com.id}"]
+  associate_public_ip_address = true
+  user_data                   = "${file("${path.module}/data/aws_launch_configuration_master-us-test-1a.masters.externallb.example.com_user_data")}"
+
+  root_block_device = {
+    volume_type           = "gp2"
+    volume_size           = 64
+    delete_on_termination = true
+  }
+
+  ephemeral_block_device = {
+    device_name  = "/dev/sdc"
+    virtual_name = "ephemeral0"
+  }
+
+  lifecycle = {
+    create_before_destroy = true
+  }
+
+  enable_monitoring = false
+}
+
+resource "aws_launch_configuration" "nodes-externallb-example-com" {
+  name_prefix                 = "nodes.externallb.example.com-"
+  image_id                    = "ami-12345678"
+  instance_type               = "t2.medium"
+  key_name                    = "${aws_key_pair.kubernetes-externallb-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
+  iam_instance_profile        = "${aws_iam_instance_profile.nodes-externallb-example-com.id}"
+  security_groups             = ["${aws_security_group.nodes-externallb-example-com.id}"]
+  associate_public_ip_address = true
+  user_data                   = "${file("${path.module}/data/aws_launch_configuration_nodes.externallb.example.com_user_data")}"
+
+  root_block_device = {
+    volume_type           = "gp2"
+    volume_size           = 128
+    delete_on_termination = true
+  }
+
+  lifecycle = {
+    create_before_destroy = true
+  }
+
+  enable_monitoring = false
+}
+
+resource "aws_route" "0-0-0-0--0" {
+  route_table_id         = "${aws_route_table.externallb-example-com.id}"
+  destination_cidr_block = "0.0.0.0/0"
+  gateway_id             = "${aws_internet_gateway.externallb-example-com.id}"
+}
+
+resource "aws_route_table" "externallb-example-com" {
+  vpc_id = "${aws_vpc.externallb-example-com.id}"
+
+  tags = {
+    KubernetesCluster                              = "externallb.example.com"
+    Name                                           = "externallb.example.com"
+    "kubernetes.io/cluster/externallb.example.com" = "owned"
+    "kubernetes.io/kops/role"                      = "public"
+  }
+}
+
+resource "aws_route_table_association" "us-test-1a-externallb-example-com" {
+  subnet_id      = "${aws_subnet.us-test-1a-externallb-example-com.id}"
+  route_table_id = "${aws_route_table.externallb-example-com.id}"
+}
+
+resource "aws_security_group" "masters-externallb-example-com" {
+  name        = "masters.externallb.example.com"
+  vpc_id      = "${aws_vpc.externallb-example-com.id}"
+  description = "Security group for masters"
+
+  tags = {
+    KubernetesCluster                              = "externallb.example.com"
+    Name                                           = "masters.externallb.example.com"
+    "kubernetes.io/cluster/externallb.example.com" = "owned"
+  }
+}
+
+resource "aws_security_group" "nodes-externallb-example-com" {
+  name        = "nodes.externallb.example.com"
+  vpc_id      = "${aws_vpc.externallb-example-com.id}"
+  description = "Security group for nodes"
+
+  tags = {
+    KubernetesCluster                              = "externallb.example.com"
+    Name                                           = "nodes.externallb.example.com"
+    "kubernetes.io/cluster/externallb.example.com" = "owned"
+  }
+}
+
+resource "aws_security_group_rule" "all-master-to-master" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-externallb-example-com.id}"
+  source_security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
+  from_port                = 0
+  to_port                  = 0
+  protocol                 = "-1"
+}
+
+resource "aws_security_group_rule" "all-master-to-node" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.nodes-externallb-example-com.id}"
+  source_security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
+  from_port                = 0
+  to_port                  = 0
+  protocol                 = "-1"
+}
+
+resource "aws_security_group_rule" "all-node-to-node" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.nodes-externallb-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
+  from_port                = 0
+  to_port                  = 0
+  protocol                 = "-1"
+}
+
+resource "aws_security_group_rule" "https-external-to-master-0-0-0-0--0" {
+  type              = "ingress"
+  security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
+  from_port         = 443
+  to_port           = 443
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "master-egress" {
+  type              = "egress"
+  security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "node-egress" {
+  type              = "egress"
+  security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-externallb-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
+  from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-externallb-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
+  from_port                = 2382
+  to_port                  = 4000
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-externallb-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
+  from_port                = 4003
+  to_port                  = 65535
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-udp-1-65535" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-externallb-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
+  from_port                = 1
+  to_port                  = 65535
+  protocol                 = "udp"
+}
+
+resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" {
+  type              = "ingress"
+  security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
+  from_port         = 22
+  to_port           = 22
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "ssh-external-to-node-0-0-0-0--0" {
+  type              = "ingress"
+  security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
+  from_port         = 22
+  to_port           = 22
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_subnet" "us-test-1a-externallb-example-com" {
+  vpc_id            = "${aws_vpc.externallb-example-com.id}"
+  cidr_block        = "172.20.32.0/19"
+  availability_zone = "us-test-1a"
+
+  tags = {
+    KubernetesCluster                              = "externallb.example.com"
+    Name                                           = "us-test-1a.externallb.example.com"
+    SubnetType                                     = "Public"
+    "kubernetes.io/cluster/externallb.example.com" = "owned"
+    "kubernetes.io/role/elb"                       = "1"
+  }
+}
+
+resource "aws_vpc" "externallb-example-com" {
+  cidr_block           = "172.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    KubernetesCluster                              = "externallb.example.com"
+    Name                                           = "externallb.example.com"
+    "kubernetes.io/cluster/externallb.example.com" = "owned"
+  }
+}
+
+resource "aws_vpc_dhcp_options" "externallb-example-com" {
+  domain_name         = "us-test-1.compute.internal"
+  domain_name_servers = ["AmazonProvidedDNS"]
+
+  tags = {
+    KubernetesCluster                              = "externallb.example.com"
+    Name                                           = "externallb.example.com"
+    "kubernetes.io/cluster/externallb.example.com" = "owned"
+  }
+}
+
+resource "aws_vpc_dhcp_options_association" "externallb-example-com" {
+  vpc_id          = "${aws_vpc.externallb-example-com.id}"
+  dhcp_options_id = "${aws_vpc_dhcp_options.externallb-example-com.id}"
+}
+
+terraform = {
+  required_version = ">= 0.9.3"
+}

upup/pkg/fi/cloudup/awstasks/BUILD.bazel

@@ -18,6 +18,10 @@ go_library(
         "ebsvolume_fitask.go",
         "elastic_ip.go",
         "elasticip_fitask.go",
+        "external_load_balancer_attachment.go",
+        "external_target_group_attachment.go",
+        "externalloadbalancerattachment_fitask.go",
+        "externaltargetgroupattachment_fitask.go",
         "iaminstanceprofile.go",
         "iaminstanceprofile_fitask.go",
         "iaminstanceprofilerole.go",

upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go

@@ -459,6 +459,7 @@ type cloudformationAutoscalingGroup struct {
 	MetricsCollection       []*cloudformationASGMetricsCollection `json:"MetricsCollection,omitempty"`
 
 	LoadBalancerNames []*cloudformation.Literal `json:"LoadBalancerNames,omitempty"`
+	TargetGroupARNs   []*cloudformation.Literal `json:"TargetGroupARNs,omitempty"`
 }
 
 func (_ *AutoscalingGroup) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *AutoscalingGroup) error {

upup/pkg/fi/cloudup/awstasks/external_load_balancer_attachment.go

@@ -0,0 +1,140 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awstasks
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"github.com/golang/glog"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
+	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
+)
+
+//go:generate fitask -type=ExternalLoadBalancerAttachment
+type ExternalLoadBalancerAttachment struct {
+	Name      *string
+	Lifecycle *fi.Lifecycle
+
+	LoadBalancerName string
+
+	AutoscalingGroup *AutoscalingGroup
+}
+
+func (e *ExternalLoadBalancerAttachment) Find(c *fi.Context) (*ExternalLoadBalancerAttachment, error) {
+	cloud := c.Cloud.(awsup.AWSCloud)
+
+	if e.LoadBalancerName == "" {
+		return nil, fmt.Errorf("InstanceGroup did not have LoadBalancerNames set")
+	}
+
+	g, err := findAutoscalingGroup(cloud, *e.AutoscalingGroup.Name)
+	if err != nil {
+		return nil, err
+	}
+	if g == nil {
+		return nil, nil
+	}
+
+	for _, name := range g.LoadBalancerNames {
+		if aws.StringValue(name) != e.LoadBalancerName {
+			continue
+		}
+
+		actual := &ExternalLoadBalancerAttachment{}
+		actual.LoadBalancerName = e.LoadBalancerName
+		actual.AutoscalingGroup = e.AutoscalingGroup
+
+		// Prevent spurious changes
+		actual.Name = e.Name // ELB attachments don't have tags
+		actual.Lifecycle = e.Lifecycle
+
+		return actual, nil
+	}
+
+	return nil, nil
+}
+
+func (e *ExternalLoadBalancerAttachment) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(e, c)
+}
+
+func (s *ExternalLoadBalancerAttachment) CheckChanges(a, e, changes *ExternalLoadBalancerAttachment) error {
+	if a == nil {
+		if e.LoadBalancerName == "" {
+			return fi.RequiredField("LoadBalancerName")
+		}
+		if e.AutoscalingGroup == nil {
+			return fi.RequiredField("AutoscalingGroup")
+		}
+	}
+	return nil
+}
+
+func (_ *ExternalLoadBalancerAttachment) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *ExternalLoadBalancerAttachment) error {
+	if e.LoadBalancerName == "" {
+		return fi.RequiredField("LoadBalancerName")
+	}
+
+	request := &autoscaling.AttachLoadBalancersInput{}
+	request.AutoScalingGroupName = e.AutoscalingGroup.Name
+	request.LoadBalancerNames = aws.StringSlice([]string{e.LoadBalancerName})
+
+	glog.V(2).Infof("Attaching autoscaling group %q to ELB %q", fi.StringValue(e.AutoscalingGroup.Name), e.LoadBalancerName)
+	_, err := t.Cloud.Autoscaling().AttachLoadBalancers(request)
+	if err != nil {
+		return fmt.Errorf("error attaching autoscaling group to ELB: %v", err)
+	}
+
+	return nil
+}
+
+type terraformExternalLoadBalancerAttachment struct {
+	ELB              *terraform.Literal `json:"elb"`
+	AutoscalingGroup *terraform.Literal `json:"autoscaling_group_name,omitempty"`
+}
+
+func (_ *ExternalLoadBalancerAttachment) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *ExternalLoadBalancerAttachment) error {
+	tf := &terraformExternalLoadBalancerAttachment{
+		ELB:              terraform.LiteralFromStringValue(e.LoadBalancerName),
+		AutoscalingGroup: e.AutoscalingGroup.TerraformLink(),
+	}
+
+	return t.RenderResource("aws_autoscaling_attachment", *e.Name, tf)
+}
+
+func (e *ExternalLoadBalancerAttachment) TerraformLink() *terraform.Literal {
+	return terraform.LiteralProperty("aws_autoscaling_attachment", e.LoadBalancerName, "id")
+}
+
+func (_ *ExternalLoadBalancerAttachment) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *ExternalLoadBalancerAttachment) error {
+	cfObj, ok := t.Find(e.AutoscalingGroup.CloudformationLink())
+	if !ok {
+		// topo-sort fail?
+		return fmt.Errorf("AutoScalingGroup not yet rendered")
+	}
+	cf, ok := cfObj.(*cloudformationAutoscalingGroup)
+	if !ok {
+		return fmt.Errorf("unexpected type for CF record: %T", cfObj)
+	}
+
+	cf.LoadBalancerNames = append(cf.LoadBalancerNames, cloudformation.LiteralString(e.LoadBalancerName))
+	return nil
+}

upup/pkg/fi/cloudup/awstasks/external_target_group_attachment.go

@@ -0,0 +1,146 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awstasks
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"github.com/golang/glog"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
+	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
+)
+
+//go:generate fitask -type=ExternalTargetGroupAttachment
+type ExternalTargetGroupAttachment struct {
+	Name      *string
+	Lifecycle *fi.Lifecycle
+
+	TargetGroupARN string
+
+	AutoscalingGroup *AutoscalingGroup
+}
+
+func (e *ExternalTargetGroupAttachment) name() string {
+	re := regexp.MustCompile("arn:.*:targetgroup/")
+	return re.ReplaceAllString(*e.Name, "")
+}
+
+func (e *ExternalTargetGroupAttachment) Find(c *fi.Context) (*ExternalTargetGroupAttachment, error) {
+	cloud := c.Cloud.(awsup.AWSCloud)
+
+	if e.TargetGroupARN == "" {
+		return nil, fmt.Errorf("InstanceGroup did not have TargetGroupARNs set")
+	}
+
+	g, err := findAutoscalingGroup(cloud, *e.AutoscalingGroup.Name)
+	if err != nil {
+		return nil, err
+	}
+	if g == nil {
+		return nil, nil
+	}
+
+	for _, name := range g.TargetGroupARNs {
+		if aws.StringValue(name) != e.TargetGroupARN {
+			continue
+		}
+
+		actual := &ExternalTargetGroupAttachment{}
+		actual.TargetGroupARN = e.TargetGroupARN
+		actual.AutoscalingGroup = e.AutoscalingGroup
+
+		// Prevent spurious changes
+		actual.Name = e.Name // ELB attachments don't have tags
+		actual.Lifecycle = e.Lifecycle
+
+		return actual, nil
+	}
+
+	return nil, nil
+}
+
+func (e *ExternalTargetGroupAttachment) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(e, c)
+}
+
+func (s *ExternalTargetGroupAttachment) CheckChanges(a, e, changes *ExternalTargetGroupAttachment) error {
+	if a == nil {
+		if e.TargetGroupARN == "" {
+			return fi.RequiredField("TargetGroupARN")
+		}
+		if e.AutoscalingGroup == nil {
+			return fi.RequiredField("AutoscalingGroup")
+		}
+	}
+	return nil
+}
+
+func (_ *ExternalTargetGroupAttachment) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *ExternalTargetGroupAttachment) error {
+	if e.TargetGroupARN == "" {
+		return fi.RequiredField("TargetGroupARN")
+	}
+
+	request := &autoscaling.AttachLoadBalancerTargetGroupsInput{}
+	request.AutoScalingGroupName = e.AutoscalingGroup.Name
+	request.TargetGroupARNs = aws.StringSlice([]string{e.TargetGroupARN})
+
+	glog.V(2).Infof("Attaching autoscaling group %q to Target Group %q", fi.StringValue(e.AutoscalingGroup.Name), e.TargetGroupARN)
+	_, err := t.Cloud.Autoscaling().AttachLoadBalancerTargetGroups(request)
+	if err != nil {
+		return fmt.Errorf("error attaching autoscaling group to ELB: %v", err)
+	}
+
+	return nil
+}
+
+type terraformExternalTargetGroupAttachment struct {
+	TargetGroupARN   *terraform.Literal `json:"alb_target_group_arn,omitempty"`
+	AutoscalingGroup *terraform.Literal `json:"autoscaling_group_name,omitempty"`
+}
+
+func (_ *ExternalTargetGroupAttachment) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *ExternalTargetGroupAttachment) error {
+	tf := &terraformExternalTargetGroupAttachment{
+		TargetGroupARN:   terraform.LiteralFromStringValue(e.TargetGroupARN),
+		AutoscalingGroup: e.AutoscalingGroup.TerraformLink(),
+	}
+
+	return t.RenderResource("aws_autoscaling_attachment", e.name(), tf)
+}
+
+func (e *ExternalTargetGroupAttachment) TerraformLink() *terraform.Literal {
+	return terraform.LiteralProperty("aws_autoscaling_attachment", e.name(), "id")
+}
+
+func (_ *ExternalTargetGroupAttachment) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *ExternalTargetGroupAttachment) error {
+	cfObj, ok := t.Find(e.AutoscalingGroup.CloudformationLink())
+	if !ok {
+		// topo-sort fail?
+		return fmt.Errorf("AutoScalingGroup not yet rendered")
+	}
+	cf, ok := cfObj.(*cloudformationAutoscalingGroup)
+	if !ok {
+		return fmt.Errorf("unexpected type for CF record: %T", cfObj)
+	}
+
+	cf.TargetGroupARNs = append(cf.TargetGroupARNs, cloudformation.LiteralString(e.TargetGroupARN))
+	return nil
+}

upup/pkg/fi/cloudup/awstasks/externalloadbalancerattachment_fitask.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by ""fitask" -type=ExternalLoadBalancerAttachment"; DO NOT EDIT
+
+package awstasks
+
+import (
+	"encoding/json"
+
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// ExternalLoadBalancerAttachment
+
+// JSON marshalling boilerplate
+type realExternalLoadBalancerAttachment ExternalLoadBalancerAttachment
+
+// UnmarshalJSON implements conversion to JSON, supporitng an alternate specification of the object as a string
+func (o *ExternalLoadBalancerAttachment) UnmarshalJSON(data []byte) error {
+	var jsonName string
+	if err := json.Unmarshal(data, &jsonName); err == nil {
+		o.Name = &jsonName
+		return nil
+	}
+
+	var r realExternalLoadBalancerAttachment
+	if err := json.Unmarshal(data, &r); err != nil {
+		return err
+	}
+	*o = ExternalLoadBalancerAttachment(r)
+	return nil
+}
+
+var _ fi.HasLifecycle = &ExternalLoadBalancerAttachment{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *ExternalLoadBalancerAttachment) GetLifecycle() *fi.Lifecycle {
+	return o.Lifecycle
+}
+
+// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
+func (o *ExternalLoadBalancerAttachment) SetLifecycle(lifecycle fi.Lifecycle) {
+	o.Lifecycle = &lifecycle
+}
+
+var _ fi.HasName = &ExternalLoadBalancerAttachment{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *ExternalLoadBalancerAttachment) GetName() *string {
+	return o.Name
+}
+
+// SetName sets the Name of the object, implementing fi.SetName
+func (o *ExternalLoadBalancerAttachment) SetName(name string) {
+	o.Name = &name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *ExternalLoadBalancerAttachment) String() string {
+	return fi.TaskAsString(o)
+}

upup/pkg/fi/cloudup/awstasks/externaltargetgroupattachment_fitask.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by ""fitask" -type=ExternalTargetGroupAttachment"; DO NOT EDIT
+
+package awstasks
+
+import (
+	"encoding/json"
+
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// ExternalTargetGroupAttachment
+
+// JSON marshalling boilerplate
+type realExternalTargetGroupAttachment ExternalTargetGroupAttachment
+
+// UnmarshalJSON implements conversion to JSON, supporitng an alternate specification of the object as a string
+func (o *ExternalTargetGroupAttachment) UnmarshalJSON(data []byte) error {
+	var jsonName string
+	if err := json.Unmarshal(data, &jsonName); err == nil {
+		o.Name = &jsonName
+		return nil
+	}
+
+	var r realExternalTargetGroupAttachment
+	if err := json.Unmarshal(data, &r); err != nil {
+		return err
+	}
+	*o = ExternalTargetGroupAttachment(r)
+	return nil
+}
+
+var _ fi.HasLifecycle = &ExternalTargetGroupAttachment{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *ExternalTargetGroupAttachment) GetLifecycle() *fi.Lifecycle {
+	return o.Lifecycle
+}
+
+// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
+func (o *ExternalTargetGroupAttachment) SetLifecycle(lifecycle fi.Lifecycle) {
+	o.Lifecycle = &lifecycle
+}
+
+var _ fi.HasName = &ExternalTargetGroupAttachment{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *ExternalTargetGroupAttachment) GetName() *string {
+	return o.Name
+}
+
+// SetName sets the Name of the object, implementing fi.SetName
+func (o *ExternalTargetGroupAttachment) SetName(name string) {
+	o.Name = &name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *ExternalTargetGroupAttachment) String() string {
+	return fi.TaskAsString(o)
+}