From 556ed2aacef0b9a2c4ef8ab1c23579f5ba8d1c15 Mon Sep 17 00:00:00 2001
From: Justin Santa Barbara <justin@fathomdb.com>
Date: Thu, 10 May 2018 14:25:39 -0400
Subject: [PATCH 1/9] Allow for "external" egress

We don't configure routing for a subnet when it is external.  In the
case when all subnets have external egress, we should not create any
route / network objects at all (e.g. no IGW)
---
 cmd/kops/integration_test.go       |  5 +++++
 pkg/apis/kops/cluster.go           |  3 +++
 pkg/apis/kops/validation/legacy.go | 11 +++++++----
 pkg/model/network.go               | 26 +++++++++++++++++++++++---
 4 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go
index 3781b2d945..a273fc0b84 100644
--- a/cmd/kops/integration_test.go
+++ b/cmd/kops/integration_test.go
@@ -141,6 +141,11 @@ func TestPrivateKopeio(t *testing.T) {
 	runTestAWS(t, "privatekopeio.example.com", "privatekopeio", "v1alpha2", true, 1, true, nil)
 }
 
+// TestUnmanaged is a test where all the subnets opt-out of route management
+func TestUnmanaged(t *testing.T) {
+	runTestAWS(t, "unmanaged.example.com", "unmanaged", "v1alpha2", true, 1)
+}
+
 // TestPrivateSharedSubnet runs the test on a configuration with private topology & shared subnets
 func TestPrivateSharedSubnet(t *testing.T) {
 	runTestAWS(t, "private-shared-subnet.example.com", "private-shared-subnet", "v1alpha2", true, 1, true, nil)
diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go
index c2a10c20b7..cd21c64402 100644
--- a/pkg/apis/kops/cluster.go
+++ b/pkg/apis/kops/cluster.go
@@ -442,6 +442,9 @@ const (
 	SubnetTypeUtility SubnetType = "Utility"
 )
 
+// EgressExternal means that egress configuration is done externally (preconfigured)
+const EgressExternal = "External"
+
 // ClusterSubnetSpec defines a subnet
 type ClusterSubnetSpec struct {
 	// Name is the name of the subnet
diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go
index 0f6b876a25..f85534ad6b 100644
--- a/pkg/apis/kops/validation/legacy.go
+++ b/pkg/apis/kops/validation/legacy.go
@@ -563,11 +563,14 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error {
 	// Egress specification support
 	{
 		for i, s := range c.Spec.Subnets {
-			fieldSubnet := fieldSpec.Child("Subnets").Index(i)
-			if s.Egress != "" && !strings.HasPrefix(s.Egress, "nat-") && !strings.HasPrefix(s.Egress, "i-") {
-				return field.Invalid(fieldSubnet.Child("Egress"), s.Egress, "egress must be of type NAT Gateway or NAT EC2 Instance")
+			if s.Egress == "" {
+				continue
 			}
-			if s.Egress != "" && !(s.Type == "Private") {
+			fieldSubnet := fieldSpec.Child("Subnets").Index(i)
+			if !strings.HasPrefix(s.Egress, "nat-") && !strings.HasPrefix(s.Egress, "i-") && s.Egress != kops.EgressExternal {
+				return field.Invalid(fieldSubnet.Child("Egress"), s.Egress, "egress must be of type NAT Gateway or NAT EC2 Instance or 'External'")
+			}
+			if s.Egress != kops.EgressExternal && s.Type != "Private" {
 				return field.Invalid(fieldSubnet.Child("Egress"), s.Egress, "egress can only be specified for Private subnets")
 			}
 		}
diff --git a/pkg/model/network.go b/pkg/model/network.go
index b605fa63a7..e175f44b8e 100644
--- a/pkg/model/network.go
+++ b/pkg/model/network.go
@@ -40,6 +40,10 @@ type zoneInfo struct {
 	PrivateSubnets []*kops.ClusterSubnetSpec
 }
 
+func isUnmanaged(subnet *kops.ClusterSubnetSpec) bool {
+	return subnet.Egress == kops.EgressExternal
+}
+
 func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	sharedVPC := b.Cluster.SharedVPC()
 	vpcName := b.ClusterName()
@@ -120,6 +124,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		// TODO: would be good to create these as shared, to verify them
 	}
 
+	allSubnetsUnmanaged := true
 	allSubnetsShared := true
 	allSubnetsSharedInZone := make(map[string]bool)
 	for i := range b.Cluster.Spec.Subnets {
@@ -134,11 +139,15 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			allSubnetsShared = false
 			allSubnetsSharedInZone[subnetSpec.Zone] = false
 		}
+
+		if !isUnmanaged(subnetSpec) {
+			allSubnetsUnmanaged = false
+		}
 	}
 
 	// We always have a public route table, though for private networks it is only used for NGWs and ELBs
 	var publicRouteTable *awstasks.RouteTable
-	{
+	if !allSubnetsUnmanaged {
 		// The internet gateway is the main entry point to the cluster.
 		igw := &awstasks.InternetGateway{
 			Name:      s(b.ClusterName()),
@@ -224,7 +233,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 
 		switch subnetSpec.Type {
 		case kops.SubnetTypePublic, kops.SubnetTypeUtility:
-			if !sharedSubnet {
+			if !sharedSubnet && !isUnmanaged(subnetSpec) {
 				c.AddTask(&awstasks.RouteTableAssociation{
 					Name:       s(subnetSpec.Name + "." + b.ClusterName()),
 					Lifecycle:  b.Lifecycle,
@@ -236,7 +245,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		case kops.SubnetTypePrivate:
 			// Private subnets get a Network Gateway, and their own route table to associate them with the network gateway
 
-			if !sharedSubnet {
+			if !sharedSubnet && !isUnmanaged(subnetSpec) {
 				// Private Subnet Route Table Associations
 				//
 				// Map the Private subnet to the Private route table
@@ -272,6 +281,17 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		egress := info.PrivateSubnets[0].Egress
 		publicIP := info.PrivateSubnets[0].PublicIP
 
+		allUnmanaged := true
+		for _, subnetSpec := range info.PrivateSubnets {
+			if !isUnmanaged(subnetSpec) {
+				allUnmanaged = false
+			}
+		}
+		if allUnmanaged {
+			glog.V(4).Infof("skipping network configuration in zone %s - all subnets unmanaged", zone)
+			continue
+		}
+
 		// Verify we don't have mixed values for egress/publicIP - the code doesn't handle it
 		for _, subnet := range info.PrivateSubnets {
 			if subnet.Egress != egress {

From f9f7eb628e10c21f574667bc14773c3d836cabe8 Mon Sep 17 00:00:00 2001
From: Justin Santa Barbara <justin@fathomdb.com>
Date: Thu, 10 May 2018 14:26:35 -0400
Subject: [PATCH 2/9] Create integration test for unmanaged networking

---
 .../update_cluster/unmanaged/id_rsa.pub       |   1 +
 .../update_cluster/unmanaged/in-v1alpha2.yaml | 110 +++
 .../update_cluster/unmanaged/kubernetes.tf    | 770 ++++++++++++++++++
 3 files changed, 881 insertions(+)
 create mode 100755 tests/integration/update_cluster/unmanaged/id_rsa.pub
 create mode 100644 tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml
 create mode 100644 tests/integration/update_cluster/unmanaged/kubernetes.tf

diff --git a/tests/integration/update_cluster/unmanaged/id_rsa.pub b/tests/integration/update_cluster/unmanaged/id_rsa.pub
new file mode 100755
index 0000000000..81cb012783
--- /dev/null
+++ b/tests/integration/update_cluster/unmanaged/id_rsa.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==
diff --git a/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml b/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml
new file mode 100644
index 0000000000..195d5e7aa2
--- /dev/null
+++ b/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml
@@ -0,0 +1,110 @@
+apiVersion: kops/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-12T04:13:14Z"
+  name: unmanaged.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/unmanaged.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: events
+  kubernetesVersion: v1.8.2
+  masterInternalName: api.internal.unmanaged.example.com
+  masterPublicName: api.unmanaged.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    weave: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+  - 0.0.0.0/0
+  topology:
+    masters: private
+    nodes: private
+  subnets:
+  - cidr: 172.20.32.0/19
+    egress: nat-a2345678
+    name: us-test-1a
+    type: Private
+    zone: us-test-1a
+  - cidr: 172.20.64.0/19
+    egress: nat-b2345678
+    name: us-test-1b
+    type: Private
+    zone: us-test-1b
+  - cidr: 172.20.4.0/22
+    name: utility-us-test-1a
+    type: Utility
+    zone: us-test-1a
+  - cidr: 172.20.8.0/22
+    name: utility-us-test-1b
+    type: Utility
+    zone: us-test-1b
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-12T04:13:15Z"
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: unmanaged.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-12T04:13:15Z"
+  name: nodes
+  labels:
+    kops.k8s.io/cluster: unmanaged.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09
+  machineType: t2.medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test-1a
+  - us-test-1b
+
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-14T15:32:41Z"
+  name: bastion
+  labels:
+    kops.k8s.io/cluster: unmanaged.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09
+  machineType: t2.micro
+  maxSize: 1
+  minSize: 1
+  role: Bastion
+  subnets:
+  - utility-us-test-1a
diff --git a/tests/integration/update_cluster/unmanaged/kubernetes.tf b/tests/integration/update_cluster/unmanaged/kubernetes.tf
new file mode 100644
index 0000000000..bb9c7214db
--- /dev/null
+++ b/tests/integration/update_cluster/unmanaged/kubernetes.tf
@@ -0,0 +1,770 @@
+output "bastion_security_group_ids" {
+  value = ["${aws_security_group.bastion-unmanaged-example-com.id}"]
+}
+
+output "bastions_role_arn" {
+  value = "${aws_iam_role.bastions-unmanaged-example-com.arn}"
+}
+
+output "bastions_role_name" {
+  value = "${aws_iam_role.bastions-unmanaged-example-com.name}"
+}
+
+output "cluster_name" {
+  value = "unmanaged.example.com"
+}
+
+output "master_security_group_ids" {
+  value = ["${aws_security_group.masters-unmanaged-example-com.id}"]
+}
+
+output "masters_role_arn" {
+  value = "${aws_iam_role.masters-unmanaged-example-com.arn}"
+}
+
+output "masters_role_name" {
+  value = "${aws_iam_role.masters-unmanaged-example-com.name}"
+}
+
+output "node_security_group_ids" {
+  value = ["${aws_security_group.nodes-unmanaged-example-com.id}"]
+}
+
+output "node_subnet_ids" {
+  value = ["${aws_subnet.us-test-1a-unmanaged-example-com.id}", "${aws_subnet.us-test-1b-unmanaged-example-com.id}"]
+}
+
+output "nodes_role_arn" {
+  value = "${aws_iam_role.nodes-unmanaged-example-com.arn}"
+}
+
+output "nodes_role_name" {
+  value = "${aws_iam_role.nodes-unmanaged-example-com.name}"
+}
+
+output "region" {
+  value = "us-test-1"
+}
+
+output "vpc_id" {
+  value = "${aws_vpc.unmanaged-example-com.id}"
+}
+
+provider "aws" {
+  region = "us-test-1"
+}
+
+resource "aws_autoscaling_attachment" "bastion-unmanaged-example-com" {
+  elb                    = "${aws_elb.bastion-unmanaged-example-com.id}"
+  autoscaling_group_name = "${aws_autoscaling_group.bastion-unmanaged-example-com.id}"
+}
+
+resource "aws_autoscaling_attachment" "master-us-test-1a-masters-unmanaged-example-com" {
+  elb                    = "${aws_elb.api-unmanaged-example-com.id}"
+  autoscaling_group_name = "${aws_autoscaling_group.master-us-test-1a-masters-unmanaged-example-com.id}"
+}
+
+resource "aws_autoscaling_group" "bastion-unmanaged-example-com" {
+  name                 = "bastion.unmanaged.example.com"
+  launch_configuration = "${aws_launch_configuration.bastion-unmanaged-example-com.id}"
+  max_size             = 1
+  min_size             = 1
+  vpc_zone_identifier  = ["${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}"]
+
+  tag = {
+    key                 = "KubernetesCluster"
+    value               = "unmanaged.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "Name"
+    value               = "bastion.unmanaged.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "k8s.io/role/bastion"
+    value               = "1"
+    propagate_at_launch = true
+  }
+
+  metrics_granularity = "1Minute"
+  enabled_metrics     = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+}
+
+resource "aws_autoscaling_group" "master-us-test-1a-masters-unmanaged-example-com" {
+  name                 = "master-us-test-1a.masters.unmanaged.example.com"
+  launch_configuration = "${aws_launch_configuration.master-us-test-1a-masters-unmanaged-example-com.id}"
+  max_size             = 1
+  min_size             = 1
+  vpc_zone_identifier  = ["${aws_subnet.us-test-1a-unmanaged-example-com.id}"]
+
+  tag = {
+    key                 = "KubernetesCluster"
+    value               = "unmanaged.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "Name"
+    value               = "master-us-test-1a.masters.unmanaged.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "k8s.io/role/master"
+    value               = "1"
+    propagate_at_launch = true
+  }
+
+  metrics_granularity = "1Minute"
+  enabled_metrics     = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+}
+
+resource "aws_autoscaling_group" "nodes-unmanaged-example-com" {
+  name                 = "nodes.unmanaged.example.com"
+  launch_configuration = "${aws_launch_configuration.nodes-unmanaged-example-com.id}"
+  max_size             = 2
+  min_size             = 2
+  vpc_zone_identifier  = ["${aws_subnet.us-test-1a-unmanaged-example-com.id}", "${aws_subnet.us-test-1b-unmanaged-example-com.id}"]
+
+  tag = {
+    key                 = "KubernetesCluster"
+    value               = "unmanaged.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "Name"
+    value               = "nodes.unmanaged.example.com"
+    propagate_at_launch = true
+  }
+
+  tag = {
+    key                 = "k8s.io/role/node"
+    value               = "1"
+    propagate_at_launch = true
+  }
+
+  metrics_granularity = "1Minute"
+  enabled_metrics     = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-events-unmanaged-example-com" {
+  availability_zone = "us-test-1a"
+  size              = 20
+  type              = "gp2"
+  encrypted         = false
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "us-test-1a.etcd-events.unmanaged.example.com"
+    "k8s.io/etcd/events"                          = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                          = "1"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-main-unmanaged-example-com" {
+  availability_zone = "us-test-1a"
+  size              = 20
+  type              = "gp2"
+  encrypted         = false
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "us-test-1a.etcd-main.unmanaged.example.com"
+    "k8s.io/etcd/main"                            = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                          = "1"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_elb" "api-unmanaged-example-com" {
+  name = "api-unmanaged-example-com-t82m6f"
+
+  listener = {
+    instance_port     = 443
+    instance_protocol = "TCP"
+    lb_port           = 443
+    lb_protocol       = "TCP"
+  }
+
+  security_groups = ["${aws_security_group.api-elb-unmanaged-example-com.id}"]
+  subnets         = ["${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}", "${aws_subnet.utility-us-test-1b-unmanaged-example-com.id}"]
+
+  health_check = {
+    target              = "SSL:443"
+    healthy_threshold   = 2
+    unhealthy_threshold = 2
+    interval            = 10
+    timeout             = 5
+  }
+
+  idle_timeout = 300
+
+  tags = {
+    KubernetesCluster = "unmanaged.example.com"
+    Name              = "api.unmanaged.example.com"
+  }
+}
+
+resource "aws_elb" "bastion-unmanaged-example-com" {
+  name = "bastion-unmanaged-example-d7bn3d"
+
+  listener = {
+    instance_port     = 22
+    instance_protocol = "TCP"
+    lb_port           = 22
+    lb_protocol       = "TCP"
+  }
+
+  security_groups = ["${aws_security_group.bastion-elb-unmanaged-example-com.id}"]
+  subnets         = ["${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}"]
+
+  health_check = {
+    target              = "TCP:22"
+    healthy_threshold   = 2
+    unhealthy_threshold = 2
+    interval            = 10
+    timeout             = 5
+  }
+
+  idle_timeout = 300
+
+  tags = {
+    KubernetesCluster = "unmanaged.example.com"
+    Name              = "bastion.unmanaged.example.com"
+  }
+}
+
+resource "aws_iam_instance_profile" "bastions-unmanaged-example-com" {
+  name = "bastions.unmanaged.example.com"
+  role = "${aws_iam_role.bastions-unmanaged-example-com.name}"
+}
+
+resource "aws_iam_instance_profile" "masters-unmanaged-example-com" {
+  name = "masters.unmanaged.example.com"
+  role = "${aws_iam_role.masters-unmanaged-example-com.name}"
+}
+
+resource "aws_iam_instance_profile" "nodes-unmanaged-example-com" {
+  name = "nodes.unmanaged.example.com"
+  role = "${aws_iam_role.nodes-unmanaged-example-com.name}"
+}
+
+resource "aws_iam_role" "bastions-unmanaged-example-com" {
+  name               = "bastions.unmanaged.example.com"
+  assume_role_policy = "${file("${path.module}/data/aws_iam_role_bastions.unmanaged.example.com_policy")}"
+}
+
+resource "aws_iam_role" "masters-unmanaged-example-com" {
+  name               = "masters.unmanaged.example.com"
+  assume_role_policy = "${file("${path.module}/data/aws_iam_role_masters.unmanaged.example.com_policy")}"
+}
+
+resource "aws_iam_role" "nodes-unmanaged-example-com" {
+  name               = "nodes.unmanaged.example.com"
+  assume_role_policy = "${file("${path.module}/data/aws_iam_role_nodes.unmanaged.example.com_policy")}"
+}
+
+resource "aws_iam_role_policy" "bastions-unmanaged-example-com" {
+  name   = "bastions.unmanaged.example.com"
+  role   = "${aws_iam_role.bastions-unmanaged-example-com.name}"
+  policy = "${file("${path.module}/data/aws_iam_role_policy_bastions.unmanaged.example.com_policy")}"
+}
+
+resource "aws_iam_role_policy" "masters-unmanaged-example-com" {
+  name   = "masters.unmanaged.example.com"
+  role   = "${aws_iam_role.masters-unmanaged-example-com.name}"
+  policy = "${file("${path.module}/data/aws_iam_role_policy_masters.unmanaged.example.com_policy")}"
+}
+
+resource "aws_iam_role_policy" "nodes-unmanaged-example-com" {
+  name   = "nodes.unmanaged.example.com"
+  role   = "${aws_iam_role.nodes-unmanaged-example-com.name}"
+  policy = "${file("${path.module}/data/aws_iam_role_policy_nodes.unmanaged.example.com_policy")}"
+}
+
+resource "aws_internet_gateway" "unmanaged-example-com" {
+  vpc_id = "${aws_vpc.unmanaged-example-com.id}"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_key_pair" "kubernetes-unmanaged-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
+  key_name   = "kubernetes.unmanaged.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
+  public_key = "${file("${path.module}/data/aws_key_pair_kubernetes.unmanaged.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")}"
+}
+
+resource "aws_launch_configuration" "bastion-unmanaged-example-com" {
+  name_prefix                 = "bastion.unmanaged.example.com-"
+  image_id                    = "ami-15000000"
+  instance_type               = "t2.micro"
+  key_name                    = "${aws_key_pair.kubernetes-unmanaged-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
+  iam_instance_profile        = "${aws_iam_instance_profile.bastions-unmanaged-example-com.id}"
+  security_groups             = ["${aws_security_group.bastion-unmanaged-example-com.id}"]
+  associate_public_ip_address = true
+
+  root_block_device = {
+    volume_type           = "gp2"
+    volume_size           = 32
+    delete_on_termination = true
+  }
+
+  lifecycle = {
+    create_before_destroy = true
+  }
+
+  enable_monitoring = false
+}
+
+resource "aws_launch_configuration" "master-us-test-1a-masters-unmanaged-example-com" {
+  name_prefix                 = "master-us-test-1a.masters.unmanaged.example.com-"
+  image_id                    = "ami-15000000"
+  instance_type               = "m3.medium"
+  key_name                    = "${aws_key_pair.kubernetes-unmanaged-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
+  iam_instance_profile        = "${aws_iam_instance_profile.masters-unmanaged-example-com.id}"
+  security_groups             = ["${aws_security_group.masters-unmanaged-example-com.id}"]
+  associate_public_ip_address = false
+  user_data                   = "${file("${path.module}/data/aws_launch_configuration_master-us-test-1a.masters.unmanaged.example.com_user_data")}"
+
+  root_block_device = {
+    volume_type           = "gp2"
+    volume_size           = 64
+    delete_on_termination = true
+  }
+
+  ephemeral_block_device = {
+    device_name  = "/dev/sdc"
+    virtual_name = "ephemeral0"
+  }
+
+  lifecycle = {
+    create_before_destroy = true
+  }
+
+  enable_monitoring = false
+}
+
+resource "aws_launch_configuration" "nodes-unmanaged-example-com" {
+  name_prefix                 = "nodes.unmanaged.example.com-"
+  image_id                    = "ami-15000000"
+  instance_type               = "t2.medium"
+  key_name                    = "${aws_key_pair.kubernetes-unmanaged-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
+  iam_instance_profile        = "${aws_iam_instance_profile.nodes-unmanaged-example-com.id}"
+  security_groups             = ["${aws_security_group.nodes-unmanaged-example-com.id}"]
+  associate_public_ip_address = false
+  user_data                   = "${file("${path.module}/data/aws_launch_configuration_nodes.unmanaged.example.com_user_data")}"
+
+  root_block_device = {
+    volume_type           = "gp2"
+    volume_size           = 128
+    delete_on_termination = true
+  }
+
+  lifecycle = {
+    create_before_destroy = true
+  }
+
+  enable_monitoring = false
+}
+
+resource "aws_route" "0-0-0-0--0" {
+  route_table_id         = "${aws_route_table.unmanaged-example-com.id}"
+  destination_cidr_block = "0.0.0.0/0"
+  gateway_id             = "${aws_internet_gateway.unmanaged-example-com.id}"
+}
+
+resource "aws_route" "private-us-test-1a-0-0-0-0--0" {
+  route_table_id         = "${aws_route_table.private-us-test-1a-unmanaged-example-com.id}"
+  destination_cidr_block = "0.0.0.0/0"
+  nat_gateway_id         = "nat-a2345678"
+}
+
+resource "aws_route" "private-us-test-1b-0-0-0-0--0" {
+  route_table_id         = "${aws_route_table.private-us-test-1b-unmanaged-example-com.id}"
+  destination_cidr_block = "0.0.0.0/0"
+  nat_gateway_id         = "nat-b2345678"
+}
+
+resource "aws_route53_record" "api-unmanaged-example-com" {
+  name = "api.unmanaged.example.com"
+  type = "A"
+
+  alias = {
+    name                   = "${aws_elb.api-unmanaged-example-com.dns_name}"
+    zone_id                = "${aws_elb.api-unmanaged-example-com.zone_id}"
+    evaluate_target_health = false
+  }
+
+  zone_id = "/hostedzone/Z1AFAKE1ZON3YO"
+}
+
+resource "aws_route_table" "private-us-test-1a-unmanaged-example-com" {
+  vpc_id = "${aws_vpc.unmanaged-example-com.id}"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "private-us-test-1a.unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+    "kubernetes.io/kops/role"                     = "private-us-test-1a"
+  }
+}
+
+resource "aws_route_table" "private-us-test-1b-unmanaged-example-com" {
+  vpc_id = "${aws_vpc.unmanaged-example-com.id}"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "private-us-test-1b.unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+    "kubernetes.io/kops/role"                     = "private-us-test-1b"
+  }
+}
+
+resource "aws_route_table" "unmanaged-example-com" {
+  vpc_id = "${aws_vpc.unmanaged-example-com.id}"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+    "kubernetes.io/kops/role"                     = "public"
+  }
+}
+
+resource "aws_route_table_association" "private-us-test-1a-unmanaged-example-com" {
+  subnet_id      = "${aws_subnet.us-test-1a-unmanaged-example-com.id}"
+  route_table_id = "${aws_route_table.private-us-test-1a-unmanaged-example-com.id}"
+}
+
+resource "aws_route_table_association" "private-us-test-1b-unmanaged-example-com" {
+  subnet_id      = "${aws_subnet.us-test-1b-unmanaged-example-com.id}"
+  route_table_id = "${aws_route_table.private-us-test-1b-unmanaged-example-com.id}"
+}
+
+resource "aws_route_table_association" "utility-us-test-1a-unmanaged-example-com" {
+  subnet_id      = "${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}"
+  route_table_id = "${aws_route_table.unmanaged-example-com.id}"
+}
+
+resource "aws_route_table_association" "utility-us-test-1b-unmanaged-example-com" {
+  subnet_id      = "${aws_subnet.utility-us-test-1b-unmanaged-example-com.id}"
+  route_table_id = "${aws_route_table.unmanaged-example-com.id}"
+}
+
+resource "aws_security_group" "api-elb-unmanaged-example-com" {
+  name        = "api-elb.unmanaged.example.com"
+  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  description = "Security group for api ELB"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "api-elb.unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_security_group" "bastion-elb-unmanaged-example-com" {
+  name        = "bastion-elb.unmanaged.example.com"
+  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  description = "Security group for bastion ELB"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "bastion-elb.unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_security_group" "bastion-unmanaged-example-com" {
+  name        = "bastion.unmanaged.example.com"
+  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  description = "Security group for bastion"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "bastion.unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_security_group" "masters-unmanaged-example-com" {
+  name        = "masters.unmanaged.example.com"
+  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  description = "Security group for masters"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "masters.unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_security_group" "nodes-unmanaged-example-com" {
+  name        = "nodes.unmanaged.example.com"
+  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  description = "Security group for nodes"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "nodes.unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_security_group_rule" "all-master-to-master" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}"
+  from_port                = 0
+  to_port                  = 0
+  protocol                 = "-1"
+}
+
+resource "aws_security_group_rule" "all-master-to-node" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.nodes-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}"
+  from_port                = 0
+  to_port                  = 0
+  protocol                 = "-1"
+}
+
+resource "aws_security_group_rule" "all-node-to-node" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.nodes-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}"
+  from_port                = 0
+  to_port                  = 0
+  protocol                 = "-1"
+}
+
+resource "aws_security_group_rule" "api-elb-egress" {
+  type              = "egress"
+  security_group_id = "${aws_security_group.api-elb-unmanaged-example-com.id}"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "bastion-egress" {
+  type              = "egress"
+  security_group_id = "${aws_security_group.bastion-unmanaged-example-com.id}"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "bastion-elb-egress" {
+  type              = "egress"
+  security_group_id = "${aws_security_group.bastion-elb-unmanaged-example-com.id}"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "bastion-to-master-ssh" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.bastion-unmanaged-example-com.id}"
+  from_port                = 22
+  to_port                  = 22
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "bastion-to-node-ssh" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.nodes-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.bastion-unmanaged-example-com.id}"
+  from_port                = 22
+  to_port                  = 22
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "https-api-elb-0-0-0-0--0" {
+  type              = "ingress"
+  security_group_id = "${aws_security_group.api-elb-unmanaged-example-com.id}"
+  from_port         = 443
+  to_port           = 443
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "https-elb-to-master" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.api-elb-unmanaged-example-com.id}"
+  from_port                = 443
+  to_port                  = 443
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "master-egress" {
+  type              = "egress"
+  security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "node-egress" {
+  type              = "egress"
+  security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}"
+  from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}"
+  from_port                = 2382
+  to_port                  = 4000
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}"
+  from_port                = 4003
+  to_port                  = 65535
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-udp-1-65535" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}"
+  from_port                = 1
+  to_port                  = 65535
+  protocol                 = "udp"
+}
+
+resource "aws_security_group_rule" "ssh-elb-to-bastion" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.bastion-unmanaged-example-com.id}"
+  source_security_group_id = "${aws_security_group.bastion-elb-unmanaged-example-com.id}"
+  from_port                = 22
+  to_port                  = 22
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "ssh-external-to-bastion-elb-0-0-0-0--0" {
+  type              = "ingress"
+  security_group_id = "${aws_security_group.bastion-elb-unmanaged-example-com.id}"
+  from_port         = 22
+  to_port           = 22
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+resource "aws_subnet" "us-test-1a-unmanaged-example-com" {
+  vpc_id            = "${aws_vpc.unmanaged-example-com.id}"
+  cidr_block        = "172.20.32.0/19"
+  availability_zone = "us-test-1a"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "us-test-1a.unmanaged.example.com"
+    SubnetType                                    = "Private"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+    "kubernetes.io/role/internal-elb"             = "1"
+  }
+}
+
+resource "aws_subnet" "us-test-1b-unmanaged-example-com" {
+  vpc_id            = "${aws_vpc.unmanaged-example-com.id}"
+  cidr_block        = "172.20.64.0/19"
+  availability_zone = "us-test-1b"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "us-test-1b.unmanaged.example.com"
+    SubnetType                                    = "Private"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+    "kubernetes.io/role/internal-elb"             = "1"
+  }
+}
+
+resource "aws_subnet" "utility-us-test-1a-unmanaged-example-com" {
+  vpc_id            = "${aws_vpc.unmanaged-example-com.id}"
+  cidr_block        = "172.20.4.0/22"
+  availability_zone = "us-test-1a"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "utility-us-test-1a.unmanaged.example.com"
+    SubnetType                                    = "Utility"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+    "kubernetes.io/role/elb"                      = "1"
+  }
+}
+
+resource "aws_subnet" "utility-us-test-1b-unmanaged-example-com" {
+  vpc_id            = "${aws_vpc.unmanaged-example-com.id}"
+  cidr_block        = "172.20.8.0/22"
+  availability_zone = "us-test-1b"
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "utility-us-test-1b.unmanaged.example.com"
+    SubnetType                                    = "Utility"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+    "kubernetes.io/role/elb"                      = "1"
+  }
+}
+
+resource "aws_vpc" "unmanaged-example-com" {
+  cidr_block           = "172.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_vpc_dhcp_options" "unmanaged-example-com" {
+  domain_name         = "us-test-1.compute.internal"
+  domain_name_servers = ["AmazonProvidedDNS"]
+
+  tags = {
+    KubernetesCluster                             = "unmanaged.example.com"
+    Name                                          = "unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
+  }
+}
+
+resource "aws_vpc_dhcp_options_association" "unmanaged-example-com" {
+  vpc_id          = "${aws_vpc.unmanaged-example-com.id}"
+  dhcp_options_id = "${aws_vpc_dhcp_options.unmanaged-example-com.id}"
+}
+
+terraform = {
+  required_version = ">= 0.9.3"
+}

From fab5f7f878c98c6cf5c817b6276b206f836ddaae Mon Sep 17 00:00:00 2001
From: Justin Santa Barbara <justin@fathomdb.com>
Date: Thu, 10 May 2018 14:57:42 -0400
Subject: [PATCH 3/9] Fix integration test to highlight changes

Split out to show the actual changes
---
 .../update_cluster/unmanaged/in-v1alpha2.yaml |   7 +-
 .../update_cluster/unmanaged/kubernetes.tf    | 129 ++----------------
 2 files changed, 15 insertions(+), 121 deletions(-)

diff --git a/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml b/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml
index 195d5e7aa2..e94446dbad 100644
--- a/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml
+++ b/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml
@@ -21,6 +21,7 @@ spec:
   kubernetesVersion: v1.8.2
   masterInternalName: api.internal.unmanaged.example.com
   masterPublicName: api.unmanaged.example.com
+  networkID: vpc-12345678
   networkCIDR: 172.20.0.0/16
   networking:
     weave: {}
@@ -32,20 +33,22 @@ spec:
     nodes: private
   subnets:
   - cidr: 172.20.32.0/19
-    egress: nat-a2345678
+    egress: External
     name: us-test-1a
     type: Private
     zone: us-test-1a
   - cidr: 172.20.64.0/19
-    egress: nat-b2345678
+    egress: External
     name: us-test-1b
     type: Private
     zone: us-test-1b
   - cidr: 172.20.4.0/22
+    egress: External
     name: utility-us-test-1a
     type: Utility
     zone: us-test-1a
   - cidr: 172.20.8.0/22
+    egress: External
     name: utility-us-test-1b
     type: Utility
     zone: us-test-1b
diff --git a/tests/integration/update_cluster/unmanaged/kubernetes.tf b/tests/integration/update_cluster/unmanaged/kubernetes.tf
index bb9c7214db..3b7bb5b970 100644
--- a/tests/integration/update_cluster/unmanaged/kubernetes.tf
+++ b/tests/integration/update_cluster/unmanaged/kubernetes.tf
@@ -47,7 +47,7 @@ output "region" {
 }
 
 output "vpc_id" {
-  value = "${aws_vpc.unmanaged-example-com.id}"
+  value = "vpc-12345678"
 }
 
 provider "aws" {
@@ -287,16 +287,6 @@ resource "aws_iam_role_policy" "nodes-unmanaged-example-com" {
   policy = "${file("${path.module}/data/aws_iam_role_policy_nodes.unmanaged.example.com_policy")}"
 }
 
-resource "aws_internet_gateway" "unmanaged-example-com" {
-  vpc_id = "${aws_vpc.unmanaged-example-com.id}"
-
-  tags = {
-    KubernetesCluster                             = "unmanaged.example.com"
-    Name                                          = "unmanaged.example.com"
-    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
-  }
-}
-
 resource "aws_key_pair" "kubernetes-unmanaged-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
   key_name   = "kubernetes.unmanaged.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
   public_key = "${file("${path.module}/data/aws_key_pair_kubernetes.unmanaged.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")}"
@@ -375,24 +365,6 @@ resource "aws_launch_configuration" "nodes-unmanaged-example-com" {
   enable_monitoring = false
 }
 
-resource "aws_route" "0-0-0-0--0" {
-  route_table_id         = "${aws_route_table.unmanaged-example-com.id}"
-  destination_cidr_block = "0.0.0.0/0"
-  gateway_id             = "${aws_internet_gateway.unmanaged-example-com.id}"
-}
-
-resource "aws_route" "private-us-test-1a-0-0-0-0--0" {
-  route_table_id         = "${aws_route_table.private-us-test-1a-unmanaged-example-com.id}"
-  destination_cidr_block = "0.0.0.0/0"
-  nat_gateway_id         = "nat-a2345678"
-}
-
-resource "aws_route" "private-us-test-1b-0-0-0-0--0" {
-  route_table_id         = "${aws_route_table.private-us-test-1b-unmanaged-example-com.id}"
-  destination_cidr_block = "0.0.0.0/0"
-  nat_gateway_id         = "nat-b2345678"
-}
-
 resource "aws_route53_record" "api-unmanaged-example-com" {
   name = "api.unmanaged.example.com"
   type = "A"
@@ -406,62 +378,9 @@ resource "aws_route53_record" "api-unmanaged-example-com" {
   zone_id = "/hostedzone/Z1AFAKE1ZON3YO"
 }
 
-resource "aws_route_table" "private-us-test-1a-unmanaged-example-com" {
-  vpc_id = "${aws_vpc.unmanaged-example-com.id}"
-
-  tags = {
-    KubernetesCluster                             = "unmanaged.example.com"
-    Name                                          = "private-us-test-1a.unmanaged.example.com"
-    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
-    "kubernetes.io/kops/role"                     = "private-us-test-1a"
-  }
-}
-
-resource "aws_route_table" "private-us-test-1b-unmanaged-example-com" {
-  vpc_id = "${aws_vpc.unmanaged-example-com.id}"
-
-  tags = {
-    KubernetesCluster                             = "unmanaged.example.com"
-    Name                                          = "private-us-test-1b.unmanaged.example.com"
-    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
-    "kubernetes.io/kops/role"                     = "private-us-test-1b"
-  }
-}
-
-resource "aws_route_table" "unmanaged-example-com" {
-  vpc_id = "${aws_vpc.unmanaged-example-com.id}"
-
-  tags = {
-    KubernetesCluster                             = "unmanaged.example.com"
-    Name                                          = "unmanaged.example.com"
-    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
-    "kubernetes.io/kops/role"                     = "public"
-  }
-}
-
-resource "aws_route_table_association" "private-us-test-1a-unmanaged-example-com" {
-  subnet_id      = "${aws_subnet.us-test-1a-unmanaged-example-com.id}"
-  route_table_id = "${aws_route_table.private-us-test-1a-unmanaged-example-com.id}"
-}
-
-resource "aws_route_table_association" "private-us-test-1b-unmanaged-example-com" {
-  subnet_id      = "${aws_subnet.us-test-1b-unmanaged-example-com.id}"
-  route_table_id = "${aws_route_table.private-us-test-1b-unmanaged-example-com.id}"
-}
-
-resource "aws_route_table_association" "utility-us-test-1a-unmanaged-example-com" {
-  subnet_id      = "${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}"
-  route_table_id = "${aws_route_table.unmanaged-example-com.id}"
-}
-
-resource "aws_route_table_association" "utility-us-test-1b-unmanaged-example-com" {
-  subnet_id      = "${aws_subnet.utility-us-test-1b-unmanaged-example-com.id}"
-  route_table_id = "${aws_route_table.unmanaged-example-com.id}"
-}
-
 resource "aws_security_group" "api-elb-unmanaged-example-com" {
   name        = "api-elb.unmanaged.example.com"
-  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  vpc_id      = "vpc-12345678"
   description = "Security group for api ELB"
 
   tags = {
@@ -473,7 +392,7 @@ resource "aws_security_group" "api-elb-unmanaged-example-com" {
 
 resource "aws_security_group" "bastion-elb-unmanaged-example-com" {
   name        = "bastion-elb.unmanaged.example.com"
-  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  vpc_id      = "vpc-12345678"
   description = "Security group for bastion ELB"
 
   tags = {
@@ -485,7 +404,7 @@ resource "aws_security_group" "bastion-elb-unmanaged-example-com" {
 
 resource "aws_security_group" "bastion-unmanaged-example-com" {
   name        = "bastion.unmanaged.example.com"
-  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  vpc_id      = "vpc-12345678"
   description = "Security group for bastion"
 
   tags = {
@@ -497,7 +416,7 @@ resource "aws_security_group" "bastion-unmanaged-example-com" {
 
 resource "aws_security_group" "masters-unmanaged-example-com" {
   name        = "masters.unmanaged.example.com"
-  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  vpc_id      = "vpc-12345678"
   description = "Security group for masters"
 
   tags = {
@@ -509,7 +428,7 @@ resource "aws_security_group" "masters-unmanaged-example-com" {
 
 resource "aws_security_group" "nodes-unmanaged-example-com" {
   name        = "nodes.unmanaged.example.com"
-  vpc_id      = "${aws_vpc.unmanaged-example-com.id}"
+  vpc_id      = "vpc-12345678"
   description = "Security group for nodes"
 
   tags = {
@@ -682,7 +601,7 @@ resource "aws_security_group_rule" "ssh-external-to-bastion-elb-0-0-0-0--0" {
 }
 
 resource "aws_subnet" "us-test-1a-unmanaged-example-com" {
-  vpc_id            = "${aws_vpc.unmanaged-example-com.id}"
+  vpc_id            = "vpc-12345678"
   cidr_block        = "172.20.32.0/19"
   availability_zone = "us-test-1a"
 
@@ -696,7 +615,7 @@ resource "aws_subnet" "us-test-1a-unmanaged-example-com" {
 }
 
 resource "aws_subnet" "us-test-1b-unmanaged-example-com" {
-  vpc_id            = "${aws_vpc.unmanaged-example-com.id}"
+  vpc_id            = "vpc-12345678"
   cidr_block        = "172.20.64.0/19"
   availability_zone = "us-test-1b"
 
@@ -710,7 +629,7 @@ resource "aws_subnet" "us-test-1b-unmanaged-example-com" {
 }
 
 resource "aws_subnet" "utility-us-test-1a-unmanaged-example-com" {
-  vpc_id            = "${aws_vpc.unmanaged-example-com.id}"
+  vpc_id            = "vpc-12345678"
   cidr_block        = "172.20.4.0/22"
   availability_zone = "us-test-1a"
 
@@ -724,7 +643,7 @@ resource "aws_subnet" "utility-us-test-1a-unmanaged-example-com" {
 }
 
 resource "aws_subnet" "utility-us-test-1b-unmanaged-example-com" {
-  vpc_id            = "${aws_vpc.unmanaged-example-com.id}"
+  vpc_id            = "vpc-12345678"
   cidr_block        = "172.20.8.0/22"
   availability_zone = "us-test-1b"
 
@@ -737,34 +656,6 @@ resource "aws_subnet" "utility-us-test-1b-unmanaged-example-com" {
   }
 }
 
-resource "aws_vpc" "unmanaged-example-com" {
-  cidr_block           = "172.20.0.0/16"
-  enable_dns_hostnames = true
-  enable_dns_support   = true
-
-  tags = {
-    KubernetesCluster                             = "unmanaged.example.com"
-    Name                                          = "unmanaged.example.com"
-    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
-  }
-}
-
-resource "aws_vpc_dhcp_options" "unmanaged-example-com" {
-  domain_name         = "us-test-1.compute.internal"
-  domain_name_servers = ["AmazonProvidedDNS"]
-
-  tags = {
-    KubernetesCluster                             = "unmanaged.example.com"
-    Name                                          = "unmanaged.example.com"
-    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
-  }
-}
-
-resource "aws_vpc_dhcp_options_association" "unmanaged-example-com" {
-  vpc_id          = "${aws_vpc.unmanaged-example-com.id}"
-  dhcp_options_id = "${aws_vpc_dhcp_options.unmanaged-example-com.id}"
-}
-
 terraform = {
   required_version = ">= 0.9.3"
 }

From 604f73d7cbfed1a718d1a03741adcfe425895851 Mon Sep 17 00:00:00 2001
From: Arjun Comar <arjun.comar@coveros.com>
Date: Wed, 31 Oct 2018 18:07:11 -0400
Subject: [PATCH 4/9] Updates to make External egress functional

---
 pkg/model/network.go                          |  2 ++
 .../jteeuwen/go-bindata/BUILD.bazel           | 20 -------------------
 .../go-bindata/go-bindata/BUILD.bazel         | 20 -------------------
 3 files changed, 2 insertions(+), 40 deletions(-)
 delete mode 100644 vendor/github.com/jteeuwen/go-bindata/BUILD.bazel
 delete mode 100644 vendor/github.com/jteeuwen/go-bindata/go-bindata/BUILD.bazel

diff --git a/pkg/model/network.go b/pkg/model/network.go
index e175f44b8e..d024a34c0f 100644
--- a/pkg/model/network.go
+++ b/pkg/model/network.go
@@ -332,6 +332,8 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 
 				c.AddTask(in)
 
+			} else if egress == "External" {
+				// Nothing to do here
 			} else {
 				return fmt.Errorf("kops currently only supports re-use of either NAT EC2 Instances or NAT Gateways. We will support more eventually! Please see https://github.com/kubernetes/kops/issues/1530")
 			}
diff --git a/vendor/github.com/jteeuwen/go-bindata/BUILD.bazel b/vendor/github.com/jteeuwen/go-bindata/BUILD.bazel
deleted file mode 100644
index da6711b52e..0000000000
--- a/vendor/github.com/jteeuwen/go-bindata/BUILD.bazel
+++ /dev/null
@@ -1,20 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "asset.go",
-        "bytewriter.go",
-        "config.go",
-        "convert.go",
-        "debug.go",
-        "doc.go",
-        "release.go",
-        "restore.go",
-        "stringwriter.go",
-        "toc.go",
-    ],
-    importmap = "k8s.io/kops/vendor/github.com/jteeuwen/go-bindata",
-    importpath = "github.com/jteeuwen/go-bindata",
-    visibility = ["//visibility:public"],
-)
diff --git a/vendor/github.com/jteeuwen/go-bindata/go-bindata/BUILD.bazel b/vendor/github.com/jteeuwen/go-bindata/go-bindata/BUILD.bazel
deleted file mode 100644
index 187dccb9d3..0000000000
--- a/vendor/github.com/jteeuwen/go-bindata/go-bindata/BUILD.bazel
+++ /dev/null
@@ -1,20 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "AppendSliceValue.go",
-        "main.go",
-        "version.go",
-    ],
-    importmap = "k8s.io/kops/vendor/github.com/jteeuwen/go-bindata/go-bindata",
-    importpath = "github.com/jteeuwen/go-bindata/go-bindata",
-    visibility = ["//visibility:private"],
-    deps = ["//vendor/github.com/jteeuwen/go-bindata:go_default_library"],
-)
-
-go_binary(
-    name = "go-bindata",
-    embed = [":go_default_library"],
-    visibility = ["//visibility:public"],
-)

From 2028841338657c498247498a52094bbf3e6c7851 Mon Sep 17 00:00:00 2001
From: Moustafa Baiou <mbaiou@vibrenthealth.com>
Date: Mon, 24 Dec 2018 13:15:26 -0500
Subject: [PATCH 5/9] fix integration test for unmanaged egress

gofmt
---
 cloudmock/aws/mockautoscaling/group.go        | 14 ++---
 cmd/kops/integration_test.go                  |  2 +-
 node-authorizer/pkg/client/helper.go          |  2 +-
 nodeup/pkg/model/kubelet.go                   |  2 +-
 pkg/model/firewall_test.go                    |  2 +-
 pkg/pki/certificate_test.go                   |  2 +-
 .../update_cluster/unmanaged/kubernetes.tf    | 51 +++++++++++++++++++
 upup/pkg/fi/cloudup/apply_cluster.go          | 20 ++++----
 upup/pkg/fi/fitasks/keypair.go                |  2 +-
 upup/pkg/fi/vfs_castore.go                    |  2 +-
 10 files changed, 75 insertions(+), 24 deletions(-)

diff --git a/cloudmock/aws/mockautoscaling/group.go b/cloudmock/aws/mockautoscaling/group.go
index a013ff43fe..3360291348 100644
--- a/cloudmock/aws/mockautoscaling/group.go
+++ b/cloudmock/aws/mockautoscaling/group.go
@@ -58,13 +58,13 @@ func (m *MockAutoscaling) CreateAutoScalingGroup(input *autoscaling.CreateAutoSc
 		DefaultCooldown:      input.DefaultCooldown,
 		DesiredCapacity:      input.DesiredCapacity,
 		// EnabledMetrics:          input.EnabledMetrics,
-		HealthCheckGracePeriod:  input.HealthCheckGracePeriod,
-		HealthCheckType:         input.HealthCheckType,
-		Instances:               []*autoscaling.Instance{},
-		LaunchConfigurationName: input.LaunchConfigurationName,
-		LoadBalancerNames:       input.LoadBalancerNames,
-		MaxSize:                 input.MaxSize,
-		MinSize:                 input.MinSize,
+		HealthCheckGracePeriod:           input.HealthCheckGracePeriod,
+		HealthCheckType:                  input.HealthCheckType,
+		Instances:                        []*autoscaling.Instance{},
+		LaunchConfigurationName:          input.LaunchConfigurationName,
+		LoadBalancerNames:                input.LoadBalancerNames,
+		MaxSize:                          input.MaxSize,
+		MinSize:                          input.MinSize,
 		NewInstancesProtectedFromScaleIn: input.NewInstancesProtectedFromScaleIn,
 		PlacementGroup:                   input.PlacementGroup,
 		// Status:                           input.Status,
diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go
index a273fc0b84..18c87f00e5 100644
--- a/cmd/kops/integration_test.go
+++ b/cmd/kops/integration_test.go
@@ -143,7 +143,7 @@ func TestPrivateKopeio(t *testing.T) {
 
 // TestUnmanaged is a test where all the subnets opt-out of route management
 func TestUnmanaged(t *testing.T) {
-	runTestAWS(t, "unmanaged.example.com", "unmanaged", "v1alpha2", true, 1)
+	runTestAWS(t, "unmanaged.example.com", "unmanaged", "v1alpha2", true, 1, true, nil)
 }
 
 // TestPrivateSharedSubnet runs the test on a configuration with private topology & shared subnets
diff --git a/node-authorizer/pkg/client/helper.go b/node-authorizer/pkg/client/helper.go
index 1c6ff87e0f..55b4262758 100644
--- a/node-authorizer/pkg/client/helper.go
+++ b/node-authorizer/pkg/client/helper.go
@@ -98,7 +98,7 @@ func makeKubeconfig(ctx context.Context, config *Config, token string) ([]byte,
 			{
 				Name: clusterName,
 				Cluster: v1.Cluster{
-					Server: config.KubeAPI,
+					Server:                   config.KubeAPI,
 					CertificateAuthorityData: content,
 				},
 			},
diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go
index a24baf16de..b01e216297 100644
--- a/nodeup/pkg/model/kubelet.go
+++ b/nodeup/pkg/model/kubelet.go
@@ -581,7 +581,7 @@ func (b *KubeletBuilder) buildMasterKubeletKubeconfig() (*nodetasks.File, error)
 
 	template := &x509.Certificate{
 		BasicConstraintsValid: true,
-		IsCA: false,
+		IsCA:                  false,
 	}
 
 	template.Subject = pkix.Name{
diff --git a/pkg/model/firewall_test.go b/pkg/model/firewall_test.go
index 7dcb74ab3f..93244a0779 100644
--- a/pkg/model/firewall_test.go
+++ b/pkg/model/firewall_test.go
@@ -64,7 +64,7 @@ func Test_SharedGroups(t *testing.T) {
 func makeTestInstanceGroupSec(role kops.InstanceGroupRole, secGroup *string) *kops.InstanceGroup {
 	return &kops.InstanceGroup{
 		Spec: kops.InstanceGroupSpec{
-			Role: role,
+			Role:                  role,
 			SecurityGroupOverride: secGroup,
 		},
 	}
diff --git a/pkg/pki/certificate_test.go b/pkg/pki/certificate_test.go
index 18a795dcb6..5e08e3dd57 100644
--- a/pkg/pki/certificate_test.go
+++ b/pkg/pki/certificate_test.go
@@ -58,7 +58,7 @@ func TestGenerateCertificate(t *testing.T) {
 		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
 		ExtKeyUsage:           []x509.ExtKeyUsage{},
 		BasicConstraintsValid: true,
-		IsCA: true,
+		IsCA:                  true,
 	}
 
 	cert, err := SignNewCertificate(key, template, nil, nil)
diff --git a/tests/integration/update_cluster/unmanaged/kubernetes.tf b/tests/integration/update_cluster/unmanaged/kubernetes.tf
index 3b7bb5b970..f1bce0ab2e 100644
--- a/tests/integration/update_cluster/unmanaged/kubernetes.tf
+++ b/tests/integration/update_cluster/unmanaged/kubernetes.tf
@@ -1,3 +1,30 @@
+locals = {
+  bastion_autoscaling_group_ids = ["${aws_autoscaling_group.bastion-unmanaged-example-com.id}"]
+  bastion_security_group_ids    = ["${aws_security_group.bastion-unmanaged-example-com.id}"]
+  bastions_role_arn             = "${aws_iam_role.bastions-unmanaged-example-com.arn}"
+  bastions_role_name            = "${aws_iam_role.bastions-unmanaged-example-com.name}"
+  cluster_name                  = "unmanaged.example.com"
+  master_autoscaling_group_ids  = ["${aws_autoscaling_group.master-us-test-1a-masters-unmanaged-example-com.id}"]
+  master_security_group_ids     = ["${aws_security_group.masters-unmanaged-example-com.id}"]
+  masters_role_arn              = "${aws_iam_role.masters-unmanaged-example-com.arn}"
+  masters_role_name             = "${aws_iam_role.masters-unmanaged-example-com.name}"
+  node_autoscaling_group_ids    = ["${aws_autoscaling_group.nodes-unmanaged-example-com.id}"]
+  node_security_group_ids       = ["${aws_security_group.nodes-unmanaged-example-com.id}"]
+  node_subnet_ids               = ["${aws_subnet.us-test-1a-unmanaged-example-com.id}", "${aws_subnet.us-test-1b-unmanaged-example-com.id}"]
+  nodes_role_arn                = "${aws_iam_role.nodes-unmanaged-example-com.arn}"
+  nodes_role_name               = "${aws_iam_role.nodes-unmanaged-example-com.name}"
+  region                        = "us-test-1"
+  subnet_us-test-1a_id          = "${aws_subnet.us-test-1a-unmanaged-example-com.id}"
+  subnet_us-test-1b_id          = "${aws_subnet.us-test-1b-unmanaged-example-com.id}"
+  subnet_utility-us-test-1a_id  = "${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}"
+  subnet_utility-us-test-1b_id  = "${aws_subnet.utility-us-test-1b-unmanaged-example-com.id}"
+  vpc_id                        = "vpc-12345678"
+}
+
+output "bastion_autoscaling_group_ids" {
+  value = ["${aws_autoscaling_group.bastion-unmanaged-example-com.id}"]
+}
+
 output "bastion_security_group_ids" {
   value = ["${aws_security_group.bastion-unmanaged-example-com.id}"]
 }
@@ -14,6 +41,10 @@ output "cluster_name" {
   value = "unmanaged.example.com"
 }
 
+output "master_autoscaling_group_ids" {
+  value = ["${aws_autoscaling_group.master-us-test-1a-masters-unmanaged-example-com.id}"]
+}
+
 output "master_security_group_ids" {
   value = ["${aws_security_group.masters-unmanaged-example-com.id}"]
 }
@@ -26,6 +57,10 @@ output "masters_role_name" {
   value = "${aws_iam_role.masters-unmanaged-example-com.name}"
 }
 
+output "node_autoscaling_group_ids" {
+  value = ["${aws_autoscaling_group.nodes-unmanaged-example-com.id}"]
+}
+
 output "node_security_group_ids" {
   value = ["${aws_security_group.nodes-unmanaged-example-com.id}"]
 }
@@ -46,6 +81,22 @@ output "region" {
   value = "us-test-1"
 }
 
+output "subnet_us-test-1a_id" {
+  value = "${aws_subnet.us-test-1a-unmanaged-example-com.id}"
+}
+
+output "subnet_us-test-1b_id" {
+  value = "${aws_subnet.us-test-1b-unmanaged-example-com.id}"
+}
+
+output "subnet_utility-us-test-1a_id" {
+  value = "${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}"
+}
+
+output "subnet_utility-us-test-1b_id" {
+  value = "${aws_subnet.utility-us-test-1b-unmanaged-example-com.id}"
+}
+
 output "vpc_id" {
   value = "vpc-12345678"
 }
diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go
index e81e37cc46..d235fb5ac3 100644
--- a/upup/pkg/fi/cloudup/apply_cluster.go
+++ b/upup/pkg/fi/cloudup/apply_cluster.go
@@ -416,16 +416,16 @@ func (c *ApplyClusterCmd) Run() error {
 				"iamRolePolicy":          &awstasks.IAMRolePolicy{},
 
 				// VPC / Networking
-				"dhcpOptions":           &awstasks.DHCPOptions{},
-				"internetGateway":       &awstasks.InternetGateway{},
-				"route":                 &awstasks.Route{},
-				"routeTable":            &awstasks.RouteTable{},
-				"routeTableAssociation": &awstasks.RouteTableAssociation{},
-				"securityGroup":         &awstasks.SecurityGroup{},
-				"securityGroupRule":     &awstasks.SecurityGroupRule{},
-				"subnet":                &awstasks.Subnet{},
-				"vpc":                   &awstasks.VPC{},
-				"ngw":                   &awstasks.NatGateway{},
+				"dhcpOptions":                &awstasks.DHCPOptions{},
+				"internetGateway":            &awstasks.InternetGateway{},
+				"route":                      &awstasks.Route{},
+				"routeTable":                 &awstasks.RouteTable{},
+				"routeTableAssociation":      &awstasks.RouteTableAssociation{},
+				"securityGroup":              &awstasks.SecurityGroup{},
+				"securityGroupRule":          &awstasks.SecurityGroupRule{},
+				"subnet":                     &awstasks.Subnet{},
+				"vpc":                        &awstasks.VPC{},
+				"ngw":                        &awstasks.NatGateway{},
 				"vpcDHDCPOptionsAssociation": &awstasks.VPCDHCPOptionsAssociation{},
 
 				// ELB
diff --git a/upup/pkg/fi/fitasks/keypair.go b/upup/pkg/fi/fitasks/keypair.go
index b6c1b47b9e..0bba132b85 100644
--- a/upup/pkg/fi/fitasks/keypair.go
+++ b/upup/pkg/fi/fitasks/keypair.go
@@ -293,7 +293,7 @@ func buildCertificateTemplateForType(certificateType string) (*x509.Certificate,
 
 	template := &x509.Certificate{
 		BasicConstraintsValid: true,
-		IsCA: false,
+		IsCA:                  false,
 	}
 
 	tokens := strings.Split(certificateType, ",")
diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go
index e73a1f423b..f519baf619 100644
--- a/upup/pkg/fi/vfs_castore.go
+++ b/upup/pkg/fi/vfs_castore.go
@@ -144,7 +144,7 @@ func BuildCAX509Template() *x509.Certificate {
 		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
 		ExtKeyUsage:           []x509.ExtKeyUsage{},
 		BasicConstraintsValid: true,
-		IsCA: true,
+		IsCA:                  true,
 	}
 	return template
 }

From b050cd7450c8579f7174b9af80e24f746e42e0ab Mon Sep 17 00:00:00 2001
From: Moustafa Baiou <mbaiou@vibrenthealth.com>
Date: Mon, 24 Dec 2018 13:59:50 -0500
Subject: [PATCH 6/9] gofmt with go@1.10

---
 cloudmock/aws/mockautoscaling/group.go | 14 +++++++-------
 node-authorizer/pkg/client/helper.go   |  2 +-
 nodeup/pkg/model/kubelet.go            |  2 +-
 pkg/model/firewall_test.go             |  2 +-
 pkg/pki/certificate_test.go            |  2 +-
 upup/pkg/fi/cloudup/apply_cluster.go   | 20 ++++++++++----------
 upup/pkg/fi/fitasks/keypair.go         |  2 +-
 upup/pkg/fi/vfs_castore.go             |  2 +-
 8 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/cloudmock/aws/mockautoscaling/group.go b/cloudmock/aws/mockautoscaling/group.go
index 3360291348..a013ff43fe 100644
--- a/cloudmock/aws/mockautoscaling/group.go
+++ b/cloudmock/aws/mockautoscaling/group.go
@@ -58,13 +58,13 @@ func (m *MockAutoscaling) CreateAutoScalingGroup(input *autoscaling.CreateAutoSc
 		DefaultCooldown:      input.DefaultCooldown,
 		DesiredCapacity:      input.DesiredCapacity,
 		// EnabledMetrics:          input.EnabledMetrics,
-		HealthCheckGracePeriod:           input.HealthCheckGracePeriod,
-		HealthCheckType:                  input.HealthCheckType,
-		Instances:                        []*autoscaling.Instance{},
-		LaunchConfigurationName:          input.LaunchConfigurationName,
-		LoadBalancerNames:                input.LoadBalancerNames,
-		MaxSize:                          input.MaxSize,
-		MinSize:                          input.MinSize,
+		HealthCheckGracePeriod:  input.HealthCheckGracePeriod,
+		HealthCheckType:         input.HealthCheckType,
+		Instances:               []*autoscaling.Instance{},
+		LaunchConfigurationName: input.LaunchConfigurationName,
+		LoadBalancerNames:       input.LoadBalancerNames,
+		MaxSize:                 input.MaxSize,
+		MinSize:                 input.MinSize,
 		NewInstancesProtectedFromScaleIn: input.NewInstancesProtectedFromScaleIn,
 		PlacementGroup:                   input.PlacementGroup,
 		// Status:                           input.Status,
diff --git a/node-authorizer/pkg/client/helper.go b/node-authorizer/pkg/client/helper.go
index 55b4262758..1c6ff87e0f 100644
--- a/node-authorizer/pkg/client/helper.go
+++ b/node-authorizer/pkg/client/helper.go
@@ -98,7 +98,7 @@ func makeKubeconfig(ctx context.Context, config *Config, token string) ([]byte,
 			{
 				Name: clusterName,
 				Cluster: v1.Cluster{
-					Server:                   config.KubeAPI,
+					Server: config.KubeAPI,
 					CertificateAuthorityData: content,
 				},
 			},
diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go
index b01e216297..a24baf16de 100644
--- a/nodeup/pkg/model/kubelet.go
+++ b/nodeup/pkg/model/kubelet.go
@@ -581,7 +581,7 @@ func (b *KubeletBuilder) buildMasterKubeletKubeconfig() (*nodetasks.File, error)
 
 	template := &x509.Certificate{
 		BasicConstraintsValid: true,
-		IsCA:                  false,
+		IsCA: false,
 	}
 
 	template.Subject = pkix.Name{
diff --git a/pkg/model/firewall_test.go b/pkg/model/firewall_test.go
index 93244a0779..7dcb74ab3f 100644
--- a/pkg/model/firewall_test.go
+++ b/pkg/model/firewall_test.go
@@ -64,7 +64,7 @@ func Test_SharedGroups(t *testing.T) {
 func makeTestInstanceGroupSec(role kops.InstanceGroupRole, secGroup *string) *kops.InstanceGroup {
 	return &kops.InstanceGroup{
 		Spec: kops.InstanceGroupSpec{
-			Role:                  role,
+			Role: role,
 			SecurityGroupOverride: secGroup,
 		},
 	}
diff --git a/pkg/pki/certificate_test.go b/pkg/pki/certificate_test.go
index 5e08e3dd57..18a795dcb6 100644
--- a/pkg/pki/certificate_test.go
+++ b/pkg/pki/certificate_test.go
@@ -58,7 +58,7 @@ func TestGenerateCertificate(t *testing.T) {
 		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
 		ExtKeyUsage:           []x509.ExtKeyUsage{},
 		BasicConstraintsValid: true,
-		IsCA:                  true,
+		IsCA: true,
 	}
 
 	cert, err := SignNewCertificate(key, template, nil, nil)
diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go
index d235fb5ac3..e81e37cc46 100644
--- a/upup/pkg/fi/cloudup/apply_cluster.go
+++ b/upup/pkg/fi/cloudup/apply_cluster.go
@@ -416,16 +416,16 @@ func (c *ApplyClusterCmd) Run() error {
 				"iamRolePolicy":          &awstasks.IAMRolePolicy{},
 
 				// VPC / Networking
-				"dhcpOptions":                &awstasks.DHCPOptions{},
-				"internetGateway":            &awstasks.InternetGateway{},
-				"route":                      &awstasks.Route{},
-				"routeTable":                 &awstasks.RouteTable{},
-				"routeTableAssociation":      &awstasks.RouteTableAssociation{},
-				"securityGroup":              &awstasks.SecurityGroup{},
-				"securityGroupRule":          &awstasks.SecurityGroupRule{},
-				"subnet":                     &awstasks.Subnet{},
-				"vpc":                        &awstasks.VPC{},
-				"ngw":                        &awstasks.NatGateway{},
+				"dhcpOptions":           &awstasks.DHCPOptions{},
+				"internetGateway":       &awstasks.InternetGateway{},
+				"route":                 &awstasks.Route{},
+				"routeTable":            &awstasks.RouteTable{},
+				"routeTableAssociation": &awstasks.RouteTableAssociation{},
+				"securityGroup":         &awstasks.SecurityGroup{},
+				"securityGroupRule":     &awstasks.SecurityGroupRule{},
+				"subnet":                &awstasks.Subnet{},
+				"vpc":                   &awstasks.VPC{},
+				"ngw":                   &awstasks.NatGateway{},
 				"vpcDHDCPOptionsAssociation": &awstasks.VPCDHCPOptionsAssociation{},
 
 				// ELB
diff --git a/upup/pkg/fi/fitasks/keypair.go b/upup/pkg/fi/fitasks/keypair.go
index 0bba132b85..b6c1b47b9e 100644
--- a/upup/pkg/fi/fitasks/keypair.go
+++ b/upup/pkg/fi/fitasks/keypair.go
@@ -293,7 +293,7 @@ func buildCertificateTemplateForType(certificateType string) (*x509.Certificate,
 
 	template := &x509.Certificate{
 		BasicConstraintsValid: true,
-		IsCA:                  false,
+		IsCA: false,
 	}
 
 	tokens := strings.Split(certificateType, ",")
diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go
index f519baf619..e73a1f423b 100644
--- a/upup/pkg/fi/vfs_castore.go
+++ b/upup/pkg/fi/vfs_castore.go
@@ -144,7 +144,7 @@ func BuildCAX509Template() *x509.Certificate {
 		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
 		ExtKeyUsage:           []x509.ExtKeyUsage{},
 		BasicConstraintsValid: true,
-		IsCA:                  true,
+		IsCA: true,
 	}
 	return template
 }

From 6f8b2b5debd98375ed1e3cc20f9fcaf287fe1f21 Mon Sep 17 00:00:00 2001
From: Moustafa Baiou <mbaiou@vibrenthealth.com>
Date: Mon, 24 Dec 2018 15:09:10 -0500
Subject: [PATCH 7/9] add back bazel files

---
 .../jteeuwen/go-bindata/BUILD.bazel           | 20 +++++++++++++++++++
 .../go-bindata/go-bindata/BUILD.bazel         | 20 +++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 vendor/github.com/jteeuwen/go-bindata/BUILD.bazel
 create mode 100644 vendor/github.com/jteeuwen/go-bindata/go-bindata/BUILD.bazel

diff --git a/vendor/github.com/jteeuwen/go-bindata/BUILD.bazel b/vendor/github.com/jteeuwen/go-bindata/BUILD.bazel
new file mode 100644
index 0000000000..da6711b52e
--- /dev/null
+++ b/vendor/github.com/jteeuwen/go-bindata/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "asset.go",
+        "bytewriter.go",
+        "config.go",
+        "convert.go",
+        "debug.go",
+        "doc.go",
+        "release.go",
+        "restore.go",
+        "stringwriter.go",
+        "toc.go",
+    ],
+    importmap = "k8s.io/kops/vendor/github.com/jteeuwen/go-bindata",
+    importpath = "github.com/jteeuwen/go-bindata",
+    visibility = ["//visibility:public"],
+)
diff --git a/vendor/github.com/jteeuwen/go-bindata/go-bindata/BUILD.bazel b/vendor/github.com/jteeuwen/go-bindata/go-bindata/BUILD.bazel
new file mode 100644
index 0000000000..187dccb9d3
--- /dev/null
+++ b/vendor/github.com/jteeuwen/go-bindata/go-bindata/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "AppendSliceValue.go",
+        "main.go",
+        "version.go",
+    ],
+    importmap = "k8s.io/kops/vendor/github.com/jteeuwen/go-bindata/go-bindata",
+    importpath = "github.com/jteeuwen/go-bindata/go-bindata",
+    visibility = ["//visibility:private"],
+    deps = ["//vendor/github.com/jteeuwen/go-bindata:go_default_library"],
+)
+
+go_binary(
+    name = "go-bindata",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

From cd95ebc11c5a806ec65ffc0d3d61b506a9313187 Mon Sep 17 00:00:00 2001
From: Moustafa Baiou <mbaiou@vibrenthealth.com>
Date: Fri, 11 Jan 2019 01:26:28 -0500
Subject: [PATCH 8/9] add documentation for external egress management

---
 docs/cluster_spec.md        | 12 ++++++++++++
 docs/run_in_existing_vpc.md | 32 ++++++++++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index b00e39ad21..fea8a8c8ff 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -155,6 +155,18 @@ spec:
     zone: us-east-1a
 ```
 
+In the case that you don't use NAT gateways or internet gateways, you can use the "External" flag for egress to force kops to ignore egress for the subnet. This can be useful when other tools are used to manage egress for the subnet such as virtual private gateways. Please note that your cluster may need to have access to the internet upon creation, so egress must be available upon initializing a cluster. This is intended for use when egress is managed external to kops, typically with an existing cluster.
+
+```
+spec:
+  subnets:
+  - cidr: 10.20.64.0/21
+    name: us-east-1a
+    egress: External
+    type: Private
+    zone: us-east-1a
+```
+
 #### publicIP
 The IP of an existing EIP that you would like to attach to the NAT gateway.
 
diff --git a/docs/run_in_existing_vpc.md b/docs/run_in_existing_vpc.md
index a4daacdc4c..e1666633e2 100644
--- a/docs/run_in_existing_vpc.md
+++ b/docs/run_in_existing_vpc.md
@@ -1,7 +1,8 @@
 ## Running in a shared VPC
 
-When launching into a shared VPC, the VPC & the Internet Gateway will be reused. By default we create a new subnet per zone,
-and a new route table, but you can also use a shared subnet (see [below](#shared-subnets)).
+When launching into a shared VPC, the VPC & the Internet Gateway will be reused. If you are not using an internet gateway
+ or NAT gateway you can tell _kops_ to ignore egress. By default we create a new subnet per zone, and a new route table, 
+ but you can also use a shared subnet (see [below](#shared-subnets)).
 
 1. Use `kops create cluster` with the `--vpc` argument for your existing VPC:
 
@@ -205,6 +206,33 @@ Please note:
 * kops won't create a route-table at all if we're not creating subnets.
 * In the example above the first subnet is using a shared NAT Gateway while the
   second one is using a shared NAT Instance
+  
+### Externally Managed Egress
+
+If you are using an unsupported egress configuration in your VPC, _kops_ can be told to ignore egress by using a configuration like:
+
+```yaml
+spec:
+  subnets:
+  - cidr: 10.20.64.0/21
+    name: us-east-1a
+    egress: External
+    type: Private
+    zone: us-east-1a
+  - cidr: 10.20.96.0/21
+    name: us-east-1b
+    egress: External
+    type: Private
+    zone: us-east-1a
+  - cidr: 10.20.32.0/21
+    name: utility-us-east-1a
+    type: Utility
+    zone: us-east-1a
+    egress: External
+```
+
+This tells _kops_ that egress is being managed externally. This is preferable when using virtual private gateways 
+(currently unsupported) or using other configurations to handle egress routing. 
 
 ### Proxy VPC Egress
 

From ec43c4a6c0e82c750dfc41f81c51fe260ea04f8e Mon Sep 17 00:00:00 2001
From: Moustafa Baiou <mbaiou@vibrenthealth.com>
Date: Sun, 27 Jan 2019 16:01:04 -0500
Subject: [PATCH 9/9] update test case

---
 tests/integration/update_cluster/unmanaged/kubernetes.tf | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/integration/update_cluster/unmanaged/kubernetes.tf b/tests/integration/update_cluster/unmanaged/kubernetes.tf
index f1bce0ab2e..1ddd23f5d3 100644
--- a/tests/integration/update_cluster/unmanaged/kubernetes.tf
+++ b/tests/integration/update_cluster/unmanaged/kubernetes.tf
@@ -579,6 +579,15 @@ resource "aws_security_group_rule" "https-elb-to-master" {
   protocol                 = "tcp"
 }
 
+resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" {
+  type              = "ingress"
+  security_group_id = "${aws_security_group.api-elb-unmanaged-example-com.id}"
+  from_port         = 3
+  to_port           = 4
+  protocol          = "icmp"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
 resource "aws_security_group_rule" "master-egress" {
   type              = "egress"
   security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}"