Support IPv6 private topology

cloudmock/aws/mockec2/BUILD.bazel

@@ -7,6 +7,7 @@ go_library(
         "api.go",
         "convenience.go",
         "dhcpoptions.go",
+        "egressonlyinternetgateways.go",
         "images.go",
         "instances.go",
         "internetgateways.go",

cloudmock/aws/mockec2/api.go

@@ -54,7 +54,8 @@ type MockEC2 struct {
 
 	Vpcs map[string]*vpcInfo
 
-	InternetGateways map[string]*ec2.InternetGateway
+	InternetGateways           map[string]*ec2.InternetGateway
+	EgressOnlyInternetGateways map[string]*ec2.EgressOnlyInternetGateway
 
 	launchTemplateNumber int
 	LaunchTemplates      map[string]*launchTemplateInfo
@@ -100,6 +101,9 @@ func (m *MockEC2) All() map[string]interface{} {
 	for id, o := range m.InternetGateways {
 		all[id] = o
 	}
+	for id, o := range m.EgressOnlyInternetGateways {
+		all[id] = o
+	}
 	for id, o := range m.LaunchTemplates {
 		all[id] = o
 	}

cloudmock/aws/mockec2/egressonlyinternetgateways.go

@@ -0,0 +1,187 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mockec2
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"k8s.io/klog/v2"
+)
+
+func (m *MockEC2) FindEgressOnlyInternetGateway(id string) *ec2.EgressOnlyInternetGateway {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	internetGateway := m.EgressOnlyInternetGateways[id]
+	if internetGateway == nil {
+		return nil
+	}
+
+	copy := *internetGateway
+	copy.Tags = m.getTags(ec2.ResourceTypeEgressOnlyInternetGateway, id)
+	return &copy
+}
+
+func (m *MockEC2) EgressOnlyInternetGatewayIds() []string {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	var ids []string
+	for id := range m.EgressOnlyInternetGateways {
+		ids = append(ids, id)
+	}
+	return ids
+}
+
+func (m *MockEC2) CreateEgressOnlyInternetGatewayRequest(*ec2.CreateEgressOnlyInternetGatewayInput) (*request.Request, *ec2.CreateEgressOnlyInternetGatewayOutput) {
+	panic("Not implemented")
+}
+
+func (m *MockEC2) CreateEgressOnlyInternetGatewayWithContext(aws.Context, *ec2.CreateEgressOnlyInternetGatewayInput, ...request.Option) (*ec2.CreateEgressOnlyInternetGatewayOutput, error) {
+	panic("Not implemented")
+}
+
+func (m *MockEC2) CreateEgressOnlyInternetGateway(request *ec2.CreateEgressOnlyInternetGatewayInput) (*ec2.CreateEgressOnlyInternetGatewayOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	klog.Infof("CreateEgressOnlyInternetGateway: %v", request)
+
+	id := m.allocateId("eigw")
+	tags := tagSpecificationsToTags(request.TagSpecifications, ec2.ResourceTypeEgressOnlyInternetGateway)
+
+	eigw := &ec2.EgressOnlyInternetGateway{
+		EgressOnlyInternetGatewayId: s(id),
+		Attachments: []*ec2.InternetGatewayAttachment{
+			{
+				VpcId: request.VpcId,
+			},
+		},
+		Tags: tags,
+	}
+
+	if m.EgressOnlyInternetGateways == nil {
+		m.EgressOnlyInternetGateways = make(map[string]*ec2.EgressOnlyInternetGateway)
+	}
+	m.EgressOnlyInternetGateways[id] = eigw
+
+	m.addTags(id, tags...)
+
+	response := &ec2.CreateEgressOnlyInternetGatewayOutput{
+		EgressOnlyInternetGateway: eigw,
+	}
+	return response, nil
+}
+
+func (m *MockEC2) DescribeEgressOnlyInternetGatewaysRequest(*ec2.DescribeEgressOnlyInternetGatewaysInput) (*request.Request, *ec2.DescribeEgressOnlyInternetGatewaysOutput) {
+	panic("Not implemented")
+}
+
+func (m *MockEC2) DescribeEgressOnlyInternetGatewaysWithContext(aws.Context, *ec2.DescribeEgressOnlyInternetGatewaysInput, ...request.Option) (*ec2.DescribeEgressOnlyInternetGatewaysOutput, error) {
+	panic("Not implemented")
+}
+
+func (m *MockEC2) DescribeEgressOnlyInternetGateways(request *ec2.DescribeEgressOnlyInternetGatewaysInput) (*ec2.DescribeEgressOnlyInternetGatewaysOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	klog.Infof("DescribeEgressOnlyInternetGateways: %v", request)
+
+	var internetGateways []*ec2.EgressOnlyInternetGateway
+
+	if len(request.EgressOnlyInternetGatewayIds) != 0 {
+		request.Filters = append(request.Filters, &ec2.Filter{Name: s("egress-only-internet-gateway-id"), Values: request.EgressOnlyInternetGatewayIds})
+	}
+
+	for id, internetGateway := range m.EgressOnlyInternetGateways {
+		allFiltersMatch := true
+		for _, filter := range request.Filters {
+			match := false
+			switch *filter.Name {
+			case "internet-gateway-id":
+				for _, v := range filter.Values {
+					if id == aws.StringValue(v) {
+						match = true
+					}
+				}
+
+			case "attachment.vpc-id":
+				for _, v := range filter.Values {
+					if internetGateway.Attachments != nil {
+						for _, attachment := range internetGateway.Attachments {
+							if *attachment.VpcId == *v {
+								match = true
+							}
+						}
+					}
+				}
+
+			default:
+				if strings.HasPrefix(*filter.Name, "tag:") {
+					match = m.hasTag(ec2.ResourceTypeEgressOnlyInternetGateway, id, filter)
+				} else {
+					return nil, fmt.Errorf("unknown filter name: %q", *filter.Name)
+				}
+			}
+
+			if !match {
+				allFiltersMatch = false
+				break
+			}
+		}
+
+		if !allFiltersMatch {
+			continue
+		}
+
+		copy := *internetGateway
+		copy.Tags = m.getTags(ec2.ResourceTypeEgressOnlyInternetGateway, id)
+		internetGateways = append(internetGateways, &copy)
+	}
+
+	response := &ec2.DescribeEgressOnlyInternetGatewaysOutput{
+		EgressOnlyInternetGateways: internetGateways,
+	}
+
+	return response, nil
+}
+func (m *MockEC2) DeleteEgressOnlyInternetGateway(request *ec2.DeleteEgressOnlyInternetGatewayInput) (*ec2.DeleteEgressOnlyInternetGatewayOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	klog.Infof("DeleteEgressOnlyInternetGateway: %v", request)
+
+	id := aws.StringValue(request.EgressOnlyInternetGatewayId)
+	o := m.EgressOnlyInternetGateways[id]
+	if o == nil {
+		return nil, fmt.Errorf("EgressOnlyInternetGateway %q not found", id)
+	}
+	delete(m.EgressOnlyInternetGateways, id)
+
+	return &ec2.DeleteEgressOnlyInternetGatewayOutput{}, nil
+}
+
+func (m *MockEC2) DeleteEgressOnlyInternetGatewayWithContext(aws.Context, *ec2.DeleteEgressOnlyInternetGatewayInput, ...request.Option) (*ec2.DeleteEgressOnlyInternetGatewayOutput, error) {
+	panic("Not implemented")
+}
+func (m *MockEC2) DeleteEgressOnlyInternetGatewayRequest(*ec2.DeleteEgressOnlyInternetGatewayInput) (*request.Request, *ec2.DeleteEgressOnlyInternetGatewayOutput) {
+	panic("Not implemented")
+}

cloudmock/aws/mockec2/tags.go

@@ -61,6 +61,8 @@ func (m *MockEC2) addTags(resourceId string, tags ...*ec2.Tag) {
 		resourceType = ec2.ResourceTypeVolume
 	} else if strings.HasPrefix(resourceId, "igw-") {
 		resourceType = ec2.ResourceTypeInternetGateway
+	} else if strings.HasPrefix(resourceId, "eigw-") {
+		resourceType = ec2.ResourceTypeEgressOnlyInternetGateway
 	} else if strings.HasPrefix(resourceId, "nat-") {
 		resourceType = ec2.ResourceTypeNatgateway
 	} else if strings.HasPrefix(resourceId, "dopt-") {

pkg/model/awsmodel/network.go

@@ -137,6 +137,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	allSubnetsUnmanaged := true
+	allPrivateSubnetsUnmanaged := true
 	allSubnetsShared := true
 	allSubnetsSharedInZone := make(map[string]bool)
 	for i := range b.Cluster.Spec.Subnets {
@@ -154,6 +155,9 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 
 		if !isUnmanaged(subnetSpec) {
 			allSubnetsUnmanaged = false
+			if subnetSpec.Type == kops.SubnetTypePrivate {
+				allPrivateSubnetsUnmanaged = false
+			}
 		}
 	}
 
@@ -299,6 +303,21 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	// Set up private route tables & egress
+
+	// The instances in the private subnet can access the IPv6 Internet by
+	// using an egress-only internet gateway.
+	var eigw *awstasks.EgressOnlyInternetGateway
+	if !allPrivateSubnetsUnmanaged && b.IsIPv6Only() {
+		eigw = &awstasks.EgressOnlyInternetGateway{
+			Name:      fi.String(b.ClusterName()),
+			Lifecycle: b.Lifecycle,
+			VPC:       b.LinkToVPC(),
+			Shared:    fi.Bool(sharedVPC),
+		}
+		eigw.Tags = b.CloudTags(*eigw.Name, *eigw.Shared)
+		c.AddTask(eigw)
+	}
+
 	for zone, info := range infoByZone {
 		if len(info.PrivateSubnets) == 0 {
 			continue
@@ -416,7 +435,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			//
 			// All private subnets will need a NGW, one per zone
 			//
-			// The instances in the private subnet can access the Internet by
+			// The instances in the private subnet can access the IPv4 Internet by
 			// using a network address translation (NAT) gateway that resides
 			// in the public subnet.
 
@@ -434,7 +453,6 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 
 		// Private Route Table
 		//
-		// The private route table that will route to the NAT Gateway
 		// We create an owned route table if we created any subnet in that zone.
 		// Otherwise we consider it shared.
 		routeTableShared := allSubnetsSharedInZone[zone]
@@ -453,7 +471,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		// Private Routes
 		//
 		// Routes for the private route table.
-		// Will route to the NAT Gateway
+		// Will route IPv4 to the NAT Gateway
 		var r *awstasks.Route
 		if in != nil {
 
@@ -479,6 +497,17 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 		c.AddTask(r)
 
+		if b.IsIPv6Only() {
+			// Route IPv6 to the Egress-only Internet Gateway.
+			c.AddTask(&awstasks.Route{
+				Name:                      fi.String("private-" + zone + "-::/0"),
+				Lifecycle:                 b.Lifecycle,
+				IPv6CIDR:                  fi.String("::/0"),
+				RouteTable:                rt,
+				EgressOnlyInternetGateway: eigw,
+			})
+		}
+
 	}
 
 	return nil

pkg/resources/aws/aws.go

@@ -69,6 +69,7 @@ func ListResourcesAWS(cloud awsup.AWSCloud, clusterName string) (map[string]*res
 		// EC2 VPC
 		ListDhcpOptions,
 		ListInternetGateways,
+		ListEgressOnlyInternetGateways,
 		ListRouteTables,
 		ListSubnets,
 		ListVPCs,
@@ -1112,6 +1113,81 @@ func DescribeInternetGatewaysIgnoreTags(cloud fi.Cloud) ([]*ec2.InternetGateway,
 	return gateways, nil
 }
 
+func DeleteEgressOnlyInternetGateway(cloud fi.Cloud, r *resources.Resource) error {
+	c := cloud.(awsup.AWSCloud)
+
+	id := r.ID
+
+	{
+		klog.V(2).Infof("Deleting EC2 EgressOnlyInternetGateway %q", id)
+		request := &ec2.DeleteEgressOnlyInternetGatewayInput{
+			EgressOnlyInternetGatewayId: &id,
+		}
+		_, err := c.EC2().DeleteEgressOnlyInternetGateway(request)
+		if err != nil {
+			if IsDependencyViolation(err) {
+				return err
+			}
+			if awsup.AWSErrorCode(err) == "InvalidEgressOnlyInternetGatewayID.NotFound" {
+				klog.Infof("Egress-only internet gateway %q not found; assuming already deleted", id)
+				return nil
+			}
+			return fmt.Errorf("error deleting EgressOnlyInternetGateway %q: %v", id, err)
+		}
+	}
+
+	return nil
+}
+
+func ListEgressOnlyInternetGateways(cloud fi.Cloud, clusterName string) ([]*resources.Resource, error) {
+	gateways, err := DescribeEgressOnlyInternetGateways(cloud)
+	if err != nil {
+		return nil, err
+	}
+
+	var resourceTrackers []*resources.Resource
+
+	for _, o := range gateways {
+		resourceTracker := &resources.Resource{
+			Name:    FindName(o.Tags),
+			ID:      aws.StringValue(o.EgressOnlyInternetGatewayId),
+			Type:    "egress-only-internet-gateway",
+			Deleter: DeleteEgressOnlyInternetGateway,
+			Shared:  HasSharedTag(ec2.ResourceTypeEgressOnlyInternetGateway+":"+aws.StringValue(o.EgressOnlyInternetGatewayId), o.Tags, clusterName),
+		}
+
+		var blocks []string
+		for _, a := range o.Attachments {
+			if aws.StringValue(a.VpcId) != "" {
+				blocks = append(blocks, "vpc:"+aws.StringValue(a.VpcId))
+			}
+		}
+		resourceTracker.Blocks = blocks
+
+		resourceTrackers = append(resourceTrackers, resourceTracker)
+	}
+
+	return resourceTrackers, nil
+}
+
+func DescribeEgressOnlyInternetGateways(cloud fi.Cloud) ([]*ec2.EgressOnlyInternetGateway, error) {
+	c := cloud.(awsup.AWSCloud)
+
+	klog.V(2).Infof("Listing EC2 EgressOnlyInternetGateways")
+	request := &ec2.DescribeEgressOnlyInternetGatewaysInput{
+		Filters: BuildEC2Filters(cloud),
+	}
+	response, err := c.EC2().DescribeEgressOnlyInternetGateways(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing EgressOnlyInternetGateway: %v", err)
+	}
+
+	var gateways []*ec2.EgressOnlyInternetGateway
+	gateways = append(gateways, response.EgressOnlyInternetGateways...)
+
+	return gateways, nil
+}
+
 func DeleteAutoScalingGroup(cloud fi.Cloud, r *resources.Resource) error {
 	c := cloud.(awsup.AWSCloud)
 

upup/pkg/fi/cloudup/awstasks/BUILD.bazel

@@ -22,6 +22,8 @@ go_library(
         "dnszone_fitask.go",
         "ebsvolume.go",
         "ebsvolume_fitask.go",
+        "egressonlyinternetgateway.go",
+        "egressonlyinternetgateway_fitask.go",
         "elastic_ip.go",
         "elasticip_fitask.go",
         "eventbridgerule.go",
@@ -120,6 +122,7 @@ go_test(
     srcs = [
         "autoscalinggroup_test.go",
         "ebsvolume_test.go",
+        "egressonlyinternetgateway_test.go",
         "elastic_ip_test.go",
         "internetgateway_test.go",
         "launchtemplate_target_cloudformation_test.go",

upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway.go

@@ -0,0 +1,228 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awstasks
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"k8s.io/klog/v2"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
+	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
+	"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
+)
+
+// +kops:fitask
+type EgressOnlyInternetGateway struct {
+	Name      *string
+	Lifecycle fi.Lifecycle
+
+	ID  *string
+	VPC *VPC
+	// Shared is set if this is a shared EgressOnlyInternetGateway
+	Shared *bool
+
+	// Tags is a map of aws tags that are added to the EgressOnlyInternetGateway
+	Tags map[string]string
+}
+
+var _ fi.CompareWithID = &EgressOnlyInternetGateway{}
+
+func (e *EgressOnlyInternetGateway) CompareWithID() *string {
+	return e.ID
+}
+
+func findEgressOnlyInternetGateway(cloud awsup.AWSCloud, request *ec2.DescribeEgressOnlyInternetGatewaysInput) (*ec2.EgressOnlyInternetGateway, error) {
+	response, err := cloud.EC2().DescribeEgressOnlyInternetGateways(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing EgressOnlyInternetGateways: %v", err)
+	}
+	if response == nil || len(response.EgressOnlyInternetGateways) == 0 {
+		return nil, nil
+	}
+
+	if len(response.EgressOnlyInternetGateways) != 1 {
+		return nil, fmt.Errorf("found multiple EgressOnlyInternetGateways matching tags")
+	}
+	igw := response.EgressOnlyInternetGateways[0]
+	return igw, nil
+}
+
+func (e *EgressOnlyInternetGateway) Find(c *fi.Context) (*EgressOnlyInternetGateway, error) {
+	cloud := c.Cloud.(awsup.AWSCloud)
+
+	request := &ec2.DescribeEgressOnlyInternetGatewaysInput{}
+
+	shared := fi.BoolValue(e.Shared)
+	if shared {
+		if fi.StringValue(e.VPC.ID) == "" {
+			return nil, fmt.Errorf("VPC ID is required when EgressOnlyInternetGateway is shared")
+		}
+
+		request.Filters = []*ec2.Filter{awsup.NewEC2Filter("attachment.vpc-id", *e.VPC.ID)}
+	} else {
+		if e.ID != nil {
+			request.EgressOnlyInternetGatewayIds = []*string{e.ID}
+		} else {
+			request.Filters = cloud.BuildFilters(e.Name)
+		}
+	}
+
+	eigw, err := findEgressOnlyInternetGateway(cloud, request)
+	if err != nil {
+		return nil, err
+	}
+	if eigw == nil {
+		return nil, nil
+	}
+	actual := &EgressOnlyInternetGateway{
+		ID:   eigw.EgressOnlyInternetGatewayId,
+		Name: findNameTag(eigw.Tags),
+		Tags: intersectTags(eigw.Tags, e.Tags),
+	}
+
+	klog.V(2).Infof("found matching EgressOnlyInternetGateway %q", *actual.ID)
+
+	for _, attachment := range eigw.Attachments {
+		actual.VPC = &VPC{ID: attachment.VpcId}
+	}
+
+	// Prevent spurious comparison failures
+	actual.Shared = e.Shared
+	actual.Lifecycle = e.Lifecycle
+	if shared {
+		actual.Name = e.Name
+	}
+	if e.ID == nil {
+		e.ID = actual.ID
+	}
+
+	// We don't set the tags for a shared EIGW
+	if fi.BoolValue(e.Shared) {
+		actual.Tags = e.Tags
+	}
+
+	return actual, nil
+}
+
+func (e *EgressOnlyInternetGateway) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(e, c)
+}
+
+func (s *EgressOnlyInternetGateway) CheckChanges(a, e, changes *EgressOnlyInternetGateway) error {
+	if a != nil {
+		if changes.VPC != nil {
+			return fi.CannotChangeField("VPC")
+		}
+	}
+
+	return nil
+}
+
+func (_ *EgressOnlyInternetGateway) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EgressOnlyInternetGateway) error {
+	shared := fi.BoolValue(e.Shared)
+	if shared {
+		// Verify the EgressOnlyInternetGateway was found and matches our required settings
+		if a == nil {
+			return fmt.Errorf("EgressOnlyInternetGateway for shared VPC was not found")
+		}
+
+		return nil
+	}
+
+	if a == nil {
+		klog.V(2).Infof("Creating EgressOnlyInternetGateway")
+
+		request := &ec2.CreateEgressOnlyInternetGatewayInput{
+			VpcId:             e.VPC.ID,
+			TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeEgressOnlyInternetGateway, e.Tags),
+		}
+
+		response, err := t.Cloud.EC2().CreateEgressOnlyInternetGateway(request)
+		if err != nil {
+			return fmt.Errorf("error creating EgressOnlyInternetGateway: %v", err)
+		}
+
+		e.ID = response.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId
+		return nil
+	}
+
+	return t.UpdateTags(*e.ID, e.Tags)
+}
+
+type terraformEgressOnlyInternetGateway struct {
+	VPCID *terraformWriter.Literal `json:"vpc_id" cty:"vpc_id"`
+	Tags  map[string]string        `json:"tags,omitempty" cty:"tags"`
+}
+
+func (_ *EgressOnlyInternetGateway) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *EgressOnlyInternetGateway) error {
+	shared := fi.BoolValue(e.Shared)
+	if shared {
+		// Not terraform owned / managed
+
+		// But ... attempt to discover the ID so TerraformLink works
+		if e.ID == nil {
+			request := &ec2.DescribeEgressOnlyInternetGatewaysInput{}
+			vpcID := fi.StringValue(e.VPC.ID)
+			if vpcID == "" {
+				return fmt.Errorf("VPC ID is required when EgressOnlyInternetGateway is shared")
+			}
+			request.Filters = []*ec2.Filter{awsup.NewEC2Filter("attachment.vpc-id", vpcID)}
+			igw, err := findEgressOnlyInternetGateway(t.Cloud.(awsup.AWSCloud), request)
+			if err != nil {
+				return err
+			}
+			if igw == nil {
+				klog.Warningf("Cannot find egress-only internet gateway for VPC %q", vpcID)
+			} else {
+				e.ID = igw.EgressOnlyInternetGatewayId
+			}
+		}
+
+		return nil
+	}
+
+	tf := &terraformEgressOnlyInternetGateway{
+		VPCID: e.VPC.TerraformLink(),
+		Tags:  e.Tags,
+	}
+
+	return t.RenderResource("aws_egress_only_internet_gateway", *e.Name, tf)
+}
+
+func (e *EgressOnlyInternetGateway) TerraformLink() *terraformWriter.Literal {
+	shared := fi.BoolValue(e.Shared)
+	if shared {
+		if e.ID == nil {
+			klog.Fatalf("ID must be set, if EgressOnlyInternetGateway is shared: %s", e)
+		}
+
+		klog.V(4).Infof("reusing existing EgressOnlyInternetGateway with id %q", *e.ID)
+		return terraformWriter.LiteralFromStringValue(*e.ID)
+	}
+
+	return terraformWriter.LiteralProperty("aws_egress_only_internet_gateway", *e.Name, "id")
+}
+
+func (_ *EgressOnlyInternetGateway) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *EgressOnlyInternetGateway) error {
+	if changes != nil {
+		klog.Warning("Egress Only Internet Gateway is not supported by the cloudformation target")
+	}
+	return nil
+}

upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway_fitask.go

@@ -0,0 +1,52 @@
+//go:build !ignore_autogenerated
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by fitask. DO NOT EDIT.
+
+package awstasks
+
+import (
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// EgressOnlyInternetGateway
+
+var _ fi.HasLifecycle = &EgressOnlyInternetGateway{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *EgressOnlyInternetGateway) GetLifecycle() fi.Lifecycle {
+	return o.Lifecycle
+}
+
+// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
+func (o *EgressOnlyInternetGateway) SetLifecycle(lifecycle fi.Lifecycle) {
+	o.Lifecycle = lifecycle
+}
+
+var _ fi.HasName = &EgressOnlyInternetGateway{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *EgressOnlyInternetGateway) GetName() *string {
+	return o.Name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *EgressOnlyInternetGateway) String() string {
+	return fi.TaskAsString(o)
+}

upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway_test.go

@@ -0,0 +1,144 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awstasks
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"k8s.io/kops/cloudmock/aws/mockec2"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+)
+
+func TestSharedEgressOnlyInternetGatewayDoesNotRename(t *testing.T) {
+	cloud := awsup.BuildMockAWSCloud("us-east-1", "abc")
+	c := &mockec2.MockEC2{}
+	cloud.MockEC2 = c
+
+	// Pre-create the vpc / subnet
+	vpc, err := c.CreateVpc(&ec2.CreateVpcInput{
+		CidrBlock: aws.String("172.20.0.0/16"),
+	})
+	if err != nil {
+		t.Fatalf("error creating test VPC: %v", err)
+	}
+	_, err = c.CreateTags(&ec2.CreateTagsInput{
+		Resources: []*string{vpc.Vpc.VpcId},
+		Tags: []*ec2.Tag{
+			{
+				Key:   aws.String("Name"),
+				Value: aws.String("ExistingVPC"),
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("error tagging test vpc: %v", err)
+	}
+
+	internetGateway, err := c.CreateEgressOnlyInternetGateway(&ec2.CreateEgressOnlyInternetGatewayInput{
+		VpcId: vpc.Vpc.VpcId,
+		TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeEgressOnlyInternetGateway, map[string]string{
+			"Name": "ExistingInternetGateway",
+		}),
+	})
+	if err != nil {
+		t.Fatalf("error creating test eigw: %v", err)
+	}
+
+	// We define a function so we can rebuild the tasks, because we modify in-place when running
+	buildTasks := func() map[string]fi.Task {
+		vpc1 := &VPC{
+			Name:      s("vpc1"),
+			Lifecycle: fi.LifecycleSync,
+			CIDR:      s("172.20.0.0/16"),
+			Tags:      map[string]string{"kubernetes.io/cluster/cluster.example.com": "shared"},
+			Shared:    fi.Bool(true),
+			ID:        vpc.Vpc.VpcId,
+		}
+		eigw1 := &EgressOnlyInternetGateway{
+			Name:      s("eigw1"),
+			Lifecycle: fi.LifecycleSync,
+			VPC:       vpc1,
+			Shared:    fi.Bool(true),
+			ID:        internetGateway.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId,
+			Tags:      make(map[string]string),
+		}
+
+		return map[string]fi.Task{
+			"eigw1": eigw1,
+			"vpc1":  vpc1,
+		}
+	}
+
+	{
+		allTasks := buildTasks()
+		eigw1 := allTasks["eigw1"].(*EgressOnlyInternetGateway)
+
+		target := &awsup.AWSAPITarget{
+			Cloud: cloud,
+		}
+
+		context, err := fi.NewContext(target, nil, cloud, nil, nil, nil, true, allTasks)
+		if err != nil {
+			t.Fatalf("error building context: %v", err)
+		}
+		defer context.Close()
+
+		if err := context.RunTasks(testRunTasksOptions); err != nil {
+			t.Fatalf("unexpected error during Run: %v", err)
+		}
+
+		if fi.StringValue(eigw1.ID) == "" {
+			t.Fatalf("ID not set after create")
+		}
+
+		if len(c.EgressOnlyInternetGatewayIds()) != 1 {
+			t.Fatalf("Expected exactly one EgressOnlyInternetGateway; found %v", c.EgressOnlyInternetGatewayIds())
+		}
+
+		actual := c.FindEgressOnlyInternetGateway(*internetGateway.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId)
+		if actual == nil {
+			t.Fatalf("EgressOnlyInternetGateway created but then not found")
+		}
+		expected := &ec2.EgressOnlyInternetGateway{
+			EgressOnlyInternetGatewayId: aws.String("eigw-1"),
+			Tags: buildTags(map[string]string{
+				"Name": "ExistingInternetGateway",
+			}),
+			Attachments: []*ec2.InternetGatewayAttachment{
+				{
+					VpcId: vpc.Vpc.VpcId,
+				},
+			},
+		}
+
+		mockec2.SortTags(expected.Tags)
+		mockec2.SortTags(actual.Tags)
+
+		if !reflect.DeepEqual(actual, expected) {
+			t.Fatalf("Unexpected EgressOnlyInternetGateway: expected=%v actual=%v", expected, actual)
+		}
+	}
+
+	{
+		allTasks := buildTasks()
+		checkNoChanges(t, cloud, allTasks)
+	}
+}

upup/pkg/fi/cloudup/awstasks/route.go

@@ -41,9 +41,10 @@ type Route struct {
 
 	// Exactly one of the below fields
 	// MUST be provided.
-	InternetGateway  *InternetGateway
+	EgressOnlyInternetGateway *EgressOnlyInternetGateway
-	NatGateway       *NatGateway
+	InternetGateway           *InternetGateway
-	TransitGatewayID *string
+	NatGateway                *NatGateway
+	TransitGatewayID          *string
 }
 
 func (e *Route) Find(c *fi.Context) (*Route, error) {
@@ -84,15 +85,18 @@ func (e *Route) Find(c *fi.Context) (*Route, error) {
 				CIDR:       r.DestinationCidrBlock,
 				IPv6CIDR:   r.DestinationIpv6CidrBlock,
 			}
+			if r.EgressOnlyInternetGatewayId != nil {
+				actual.EgressOnlyInternetGateway = &EgressOnlyInternetGateway{ID: r.EgressOnlyInternetGatewayId}
+			}
 			if r.GatewayId != nil {
 				actual.InternetGateway = &InternetGateway{ID: r.GatewayId}
 			}
-			if r.NatGatewayId != nil {
-				actual.NatGateway = &NatGateway{ID: r.NatGatewayId}
-			}
 			if r.InstanceId != nil {
 				actual.Instance = &Instance{ID: r.InstanceId}
 			}
+			if r.NatGatewayId != nil {
+				actual.NatGateway = &NatGateway{ID: r.NatGatewayId}
+			}
 			if r.TransitGatewayId != nil {
 				actual.TransitGatewayID = r.TransitGatewayId
 			}
@@ -130,9 +134,15 @@ func (s *Route) CheckChanges(a, e, changes *Route) error {
 			return fi.RequiredField("CIDR/IPv6CIDR")
 		}
 		if e.CIDR != nil && e.IPv6CIDR != nil {
-			return fmt.Errorf("cannot set more than 1 CIDR or IPv6CIDR")
+			return fmt.Errorf("cannot set more than one CIDR or IPv6CIDR")
 		}
 		targetCount := 0
+		if e.EgressOnlyInternetGateway != nil {
+			targetCount++
+			if e.CIDR != nil {
+				return fmt.Errorf("cannot route IPv4 to an EgressOnlyInternetGateway")
+			}
+		}
 		if e.InternetGateway != nil {
 			targetCount++
 		}
@@ -146,10 +156,10 @@ func (s *Route) CheckChanges(a, e, changes *Route) error {
 			targetCount++
 		}
 		if targetCount == 0 {
-			return fmt.Errorf("InternetGateway, Instance, NatGateway, or TransitGateway is required")
+			return fmt.Errorf("EgressOnlyInternetGateway, InternetGateway, Instance, NatGateway, or TransitGateway is required")
 		}
 		if targetCount != 1 {
-			return fmt.Errorf("cannot set more than 1 InternetGateway, Instance, NatGateway, or TransitGateway")
+			return fmt.Errorf("cannot set more than one EgressOnlyInternetGateway, InternetGateway, Instance, NatGateway, or TransitGateway")
 		}
 	}
 
@@ -179,8 +189,10 @@ func (_ *Route) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Route) error {
 			klog.Fatal("both CIDR and IPv6CIDR were unexpectedly nil")
 		}
 
-		if e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil {
+		if e.EgressOnlyInternetGateway == nil && e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil {
 			return fmt.Errorf("missing target for route")
+		} else if e.EgressOnlyInternetGateway != nil {
+			request.EgressOnlyInternetGatewayId = checkNotNil(e.EgressOnlyInternetGateway.ID)
 		} else if e.InternetGateway != nil {
 			request.GatewayId = checkNotNil(e.InternetGateway.ID)
 		} else if e.NatGateway != nil {
@@ -259,13 +271,14 @@ func checkNotNil(s *string) *string {
 }
 
 type terraformRoute struct {
-	RouteTableID      *terraformWriter.Literal `json:"route_table_id" cty:"route_table_id"`
+	RouteTableID                *terraformWriter.Literal `json:"route_table_id" cty:"route_table_id"`
-	CIDR              *string                  `json:"destination_cidr_block,omitempty" cty:"destination_cidr_block"`
+	CIDR                        *string                  `json:"destination_cidr_block,omitempty" cty:"destination_cidr_block"`
-	IPv6CIDR          *string                  `json:"destination_ipv6_cidr_block,omitempty" cty:"destination_ipv6_cidr_block"`
+	IPv6CIDR                    *string                  `json:"destination_ipv6_cidr_block,omitempty" cty:"destination_ipv6_cidr_block"`
-	InternetGatewayID *terraformWriter.Literal `json:"gateway_id,omitempty" cty:"gateway_id"`
+	EgressOnlyInternetGatewayID *terraformWriter.Literal `json:"egress_onlygateway_id,omitempty" cty:"egress_only_gateway_id"`
-	NATGatewayID      *terraformWriter.Literal `json:"nat_gateway_id,omitempty" cty:"nat_gateway_id"`
+	InternetGatewayID           *terraformWriter.Literal `json:"gateway_id,omitempty" cty:"gateway_id"`
-	TransitGatewayID  *string                  `json:"transit_gateway_id,omitempty" cty:"transit_gateway_id"`
+	NATGatewayID                *terraformWriter.Literal `json:"nat_gateway_id,omitempty" cty:"nat_gateway_id"`
-	InstanceID        *terraformWriter.Literal `json:"instance_id,omitempty" cty:"instance_id"`
+	TransitGatewayID            *string                  `json:"transit_gateway_id,omitempty" cty:"transit_gateway_id"`
+	InstanceID                  *terraformWriter.Literal `json:"instance_id,omitempty" cty:"instance_id"`
 }
 
 func (_ *Route) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Route) error {
@@ -275,8 +288,10 @@ func (_ *Route) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Rou
 		IPv6CIDR:     e.IPv6CIDR,
 	}
 
-	if e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil {
+	if e.EgressOnlyInternetGateway == nil && e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil {
 		return fmt.Errorf("missing target for route")
+	} else if e.EgressOnlyInternetGateway != nil {
+		tf.EgressOnlyInternetGatewayID = e.EgressOnlyInternetGateway.TerraformLink()
 	} else if e.InternetGateway != nil {
 		tf.InternetGatewayID = e.InternetGateway.TerraformLink()
 	} else if e.NatGateway != nil {

upup/pkg/fi/cloudup/new_cluster.go

@@ -984,19 +984,6 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S
 			cluster.Spec.Subnets[i].Type = api.SubnetTypePublic
 		}
 
-		if opt.IPv6 {
-			cluster.Spec.NonMasqueradeCIDR = "::/0"
-			cluster.Spec.ExternalCloudControllerManager = &api.CloudControllerManagerConfig{}
-			if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS {
-				klog.Warningf("IPv6 support is EXPERIMENTAL and can be changed or removed at any time in the future!!!")
-				for i := range cluster.Spec.Subnets {
-					cluster.Spec.Subnets[i].IPv6CIDR = fmt.Sprintf("/64#%x", i)
-				}
-			} else {
-				klog.Errorf("IPv6 support is available only on AWS")
-			}
-		}
-
 	case api.TopologyPrivate:
 		if cluster.Spec.Networking.Kubenet != nil {
 			return nil, fmt.Errorf("invalid networking option %s. Kubenet does not support private topology", opt.Networking)
@@ -1085,6 +1072,19 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S
 		return nil, fmt.Errorf("invalid topology %s", opt.Topology)
 	}
 
+	if opt.IPv6 {
+		cluster.Spec.NonMasqueradeCIDR = "::/0"
+		cluster.Spec.ExternalCloudControllerManager = &api.CloudControllerManagerConfig{}
+		if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS {
+			klog.Warningf("IPv6 support is EXPERIMENTAL and can be changed or removed at any time in the future!!!")
+			for i := range cluster.Spec.Subnets {
+				cluster.Spec.Subnets[i].IPv6CIDR = fmt.Sprintf("/64#%x", i)
+			}
+		} else {
+			klog.Errorf("IPv6 support is available only on AWS")
+		}
+	}
+
 	cluster.Spec.Topology.DNS = &api.DNSSpec{}
 	switch strings.ToLower(opt.DNSType) {
 	case "public", "":