diff --git a/cloudmock/aws/mockec2/BUILD.bazel b/cloudmock/aws/mockec2/BUILD.bazel index 21ce36f750..4220593d4e 100644 --- a/cloudmock/aws/mockec2/BUILD.bazel +++ b/cloudmock/aws/mockec2/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "api.go", "convenience.go", "dhcpoptions.go", + "egressonlyinternetgateways.go", "images.go", "instances.go", "internetgateways.go", diff --git a/cloudmock/aws/mockec2/api.go b/cloudmock/aws/mockec2/api.go index 2e0ea076dc..40bdc65fd9 100644 --- a/cloudmock/aws/mockec2/api.go +++ b/cloudmock/aws/mockec2/api.go @@ -54,7 +54,8 @@ type MockEC2 struct { Vpcs map[string]*vpcInfo - InternetGateways map[string]*ec2.InternetGateway + InternetGateways map[string]*ec2.InternetGateway + EgressOnlyInternetGateways map[string]*ec2.EgressOnlyInternetGateway launchTemplateNumber int LaunchTemplates map[string]*launchTemplateInfo @@ -100,6 +101,9 @@ func (m *MockEC2) All() map[string]interface{} { for id, o := range m.InternetGateways { all[id] = o } + for id, o := range m.EgressOnlyInternetGateways { + all[id] = o + } for id, o := range m.LaunchTemplates { all[id] = o } diff --git a/cloudmock/aws/mockec2/egressonlyinternetgateways.go b/cloudmock/aws/mockec2/egressonlyinternetgateways.go new file mode 100644 index 0000000000..0d13eab85e --- /dev/null +++ b/cloudmock/aws/mockec2/egressonlyinternetgateways.go @@ -0,0 +1,187 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mockec2 + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/ec2" + "k8s.io/klog/v2" +) + +func (m *MockEC2) FindEgressOnlyInternetGateway(id string) *ec2.EgressOnlyInternetGateway { + m.mutex.Lock() + defer m.mutex.Unlock() + + internetGateway := m.EgressOnlyInternetGateways[id] + if internetGateway == nil { + return nil + } + + copy := *internetGateway + copy.Tags = m.getTags(ec2.ResourceTypeEgressOnlyInternetGateway, id) + return © +} + +func (m *MockEC2) EgressOnlyInternetGatewayIds() []string { + m.mutex.Lock() + defer m.mutex.Unlock() + + var ids []string + for id := range m.EgressOnlyInternetGateways { + ids = append(ids, id) + } + return ids +} + +func (m *MockEC2) CreateEgressOnlyInternetGatewayRequest(*ec2.CreateEgressOnlyInternetGatewayInput) (*request.Request, *ec2.CreateEgressOnlyInternetGatewayOutput) { + panic("Not implemented") +} + +func (m *MockEC2) CreateEgressOnlyInternetGatewayWithContext(aws.Context, *ec2.CreateEgressOnlyInternetGatewayInput, ...request.Option) (*ec2.CreateEgressOnlyInternetGatewayOutput, error) { + panic("Not implemented") +} + +func (m *MockEC2) CreateEgressOnlyInternetGateway(request *ec2.CreateEgressOnlyInternetGatewayInput) (*ec2.CreateEgressOnlyInternetGatewayOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + klog.Infof("CreateEgressOnlyInternetGateway: %v", request) + + id := m.allocateId("eigw") + tags := tagSpecificationsToTags(request.TagSpecifications, ec2.ResourceTypeEgressOnlyInternetGateway) + + eigw := &ec2.EgressOnlyInternetGateway{ + EgressOnlyInternetGatewayId: s(id), + Attachments: []*ec2.InternetGatewayAttachment{ + { + VpcId: request.VpcId, + }, + }, + Tags: tags, + } + + if m.EgressOnlyInternetGateways == nil { + m.EgressOnlyInternetGateways = make(map[string]*ec2.EgressOnlyInternetGateway) + } + m.EgressOnlyInternetGateways[id] = eigw + + m.addTags(id, tags...) + + response := &ec2.CreateEgressOnlyInternetGatewayOutput{ + EgressOnlyInternetGateway: eigw, + } + return response, nil +} + +func (m *MockEC2) DescribeEgressOnlyInternetGatewaysRequest(*ec2.DescribeEgressOnlyInternetGatewaysInput) (*request.Request, *ec2.DescribeEgressOnlyInternetGatewaysOutput) { + panic("Not implemented") +} + +func (m *MockEC2) DescribeEgressOnlyInternetGatewaysWithContext(aws.Context, *ec2.DescribeEgressOnlyInternetGatewaysInput, ...request.Option) (*ec2.DescribeEgressOnlyInternetGatewaysOutput, error) { + panic("Not implemented") +} + +func (m *MockEC2) DescribeEgressOnlyInternetGateways(request *ec2.DescribeEgressOnlyInternetGatewaysInput) (*ec2.DescribeEgressOnlyInternetGatewaysOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + klog.Infof("DescribeEgressOnlyInternetGateways: %v", request) + + var internetGateways []*ec2.EgressOnlyInternetGateway + + if len(request.EgressOnlyInternetGatewayIds) != 0 { + request.Filters = append(request.Filters, &ec2.Filter{Name: s("egress-only-internet-gateway-id"), Values: request.EgressOnlyInternetGatewayIds}) + } + + for id, internetGateway := range m.EgressOnlyInternetGateways { + allFiltersMatch := true + for _, filter := range request.Filters { + match := false + switch *filter.Name { + case "internet-gateway-id": + for _, v := range filter.Values { + if id == aws.StringValue(v) { + match = true + } + } + + case "attachment.vpc-id": + for _, v := range filter.Values { + if internetGateway.Attachments != nil { + for _, attachment := range internetGateway.Attachments { + if *attachment.VpcId == *v { + match = true + } + } + } + } + + default: + if strings.HasPrefix(*filter.Name, "tag:") { + match = m.hasTag(ec2.ResourceTypeEgressOnlyInternetGateway, id, filter) + } else { + return nil, fmt.Errorf("unknown filter name: %q", *filter.Name) + } + } + + if !match { + allFiltersMatch = false + break + } + } + + if !allFiltersMatch { + continue + } + + copy := *internetGateway + copy.Tags = m.getTags(ec2.ResourceTypeEgressOnlyInternetGateway, id) + internetGateways = append(internetGateways, ©) + } + + response := &ec2.DescribeEgressOnlyInternetGatewaysOutput{ + EgressOnlyInternetGateways: internetGateways, + } + + return response, nil +} +func (m *MockEC2) DeleteEgressOnlyInternetGateway(request *ec2.DeleteEgressOnlyInternetGatewayInput) (*ec2.DeleteEgressOnlyInternetGatewayOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + klog.Infof("DeleteEgressOnlyInternetGateway: %v", request) + + id := aws.StringValue(request.EgressOnlyInternetGatewayId) + o := m.EgressOnlyInternetGateways[id] + if o == nil { + return nil, fmt.Errorf("EgressOnlyInternetGateway %q not found", id) + } + delete(m.EgressOnlyInternetGateways, id) + + return &ec2.DeleteEgressOnlyInternetGatewayOutput{}, nil +} + +func (m *MockEC2) DeleteEgressOnlyInternetGatewayWithContext(aws.Context, *ec2.DeleteEgressOnlyInternetGatewayInput, ...request.Option) (*ec2.DeleteEgressOnlyInternetGatewayOutput, error) { + panic("Not implemented") +} +func (m *MockEC2) DeleteEgressOnlyInternetGatewayRequest(*ec2.DeleteEgressOnlyInternetGatewayInput) (*request.Request, *ec2.DeleteEgressOnlyInternetGatewayOutput) { + panic("Not implemented") +} diff --git a/cloudmock/aws/mockec2/tags.go b/cloudmock/aws/mockec2/tags.go index 832244edf6..0afeb53659 100644 --- a/cloudmock/aws/mockec2/tags.go +++ b/cloudmock/aws/mockec2/tags.go @@ -61,6 +61,8 @@ func (m *MockEC2) addTags(resourceId string, tags ...*ec2.Tag) { resourceType = ec2.ResourceTypeVolume } else if strings.HasPrefix(resourceId, "igw-") { resourceType = ec2.ResourceTypeInternetGateway + } else if strings.HasPrefix(resourceId, "eigw-") { + resourceType = ec2.ResourceTypeEgressOnlyInternetGateway } else if strings.HasPrefix(resourceId, "nat-") { resourceType = ec2.ResourceTypeNatgateway } else if strings.HasPrefix(resourceId, "dopt-") { diff --git a/pkg/model/awsmodel/network.go b/pkg/model/awsmodel/network.go index 2353aba810..7d8b50114c 100644 --- a/pkg/model/awsmodel/network.go +++ b/pkg/model/awsmodel/network.go @@ -137,6 +137,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { } allSubnetsUnmanaged := true + allPrivateSubnetsUnmanaged := true allSubnetsShared := true allSubnetsSharedInZone := make(map[string]bool) for i := range b.Cluster.Spec.Subnets { @@ -154,6 +155,9 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { if !isUnmanaged(subnetSpec) { allSubnetsUnmanaged = false + if subnetSpec.Type == kops.SubnetTypePrivate { + allPrivateSubnetsUnmanaged = false + } } } @@ -299,6 +303,21 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { } // Set up private route tables & egress + + // The instances in the private subnet can access the IPv6 Internet by + // using an egress-only internet gateway. + var eigw *awstasks.EgressOnlyInternetGateway + if !allPrivateSubnetsUnmanaged && b.IsIPv6Only() { + eigw = &awstasks.EgressOnlyInternetGateway{ + Name: fi.String(b.ClusterName()), + Lifecycle: b.Lifecycle, + VPC: b.LinkToVPC(), + Shared: fi.Bool(sharedVPC), + } + eigw.Tags = b.CloudTags(*eigw.Name, *eigw.Shared) + c.AddTask(eigw) + } + for zone, info := range infoByZone { if len(info.PrivateSubnets) == 0 { continue @@ -416,7 +435,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { // // All private subnets will need a NGW, one per zone // - // The instances in the private subnet can access the Internet by + // The instances in the private subnet can access the IPv4 Internet by // using a network address translation (NAT) gateway that resides // in the public subnet. @@ -434,7 +453,6 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { // Private Route Table // - // The private route table that will route to the NAT Gateway // We create an owned route table if we created any subnet in that zone. // Otherwise we consider it shared. routeTableShared := allSubnetsSharedInZone[zone] @@ -453,7 +471,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { // Private Routes // // Routes for the private route table. - // Will route to the NAT Gateway + // Will route IPv4 to the NAT Gateway var r *awstasks.Route if in != nil { @@ -479,6 +497,17 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { } c.AddTask(r) + if b.IsIPv6Only() { + // Route IPv6 to the Egress-only Internet Gateway. + c.AddTask(&awstasks.Route{ + Name: fi.String("private-" + zone + "-::/0"), + Lifecycle: b.Lifecycle, + IPv6CIDR: fi.String("::/0"), + RouteTable: rt, + EgressOnlyInternetGateway: eigw, + }) + } + } return nil diff --git a/pkg/resources/aws/aws.go b/pkg/resources/aws/aws.go index 0d16a8a15a..ced553d4aa 100644 --- a/pkg/resources/aws/aws.go +++ b/pkg/resources/aws/aws.go @@ -69,6 +69,7 @@ func ListResourcesAWS(cloud awsup.AWSCloud, clusterName string) (map[string]*res // EC2 VPC ListDhcpOptions, ListInternetGateways, + ListEgressOnlyInternetGateways, ListRouteTables, ListSubnets, ListVPCs, @@ -1112,6 +1113,81 @@ func DescribeInternetGatewaysIgnoreTags(cloud fi.Cloud) ([]*ec2.InternetGateway, return gateways, nil } +func DeleteEgressOnlyInternetGateway(cloud fi.Cloud, r *resources.Resource) error { + c := cloud.(awsup.AWSCloud) + + id := r.ID + + { + klog.V(2).Infof("Deleting EC2 EgressOnlyInternetGateway %q", id) + request := &ec2.DeleteEgressOnlyInternetGatewayInput{ + EgressOnlyInternetGatewayId: &id, + } + _, err := c.EC2().DeleteEgressOnlyInternetGateway(request) + if err != nil { + if IsDependencyViolation(err) { + return err + } + if awsup.AWSErrorCode(err) == "InvalidEgressOnlyInternetGatewayID.NotFound" { + klog.Infof("Egress-only internet gateway %q not found; assuming already deleted", id) + return nil + } + return fmt.Errorf("error deleting EgressOnlyInternetGateway %q: %v", id, err) + } + } + + return nil +} + +func ListEgressOnlyInternetGateways(cloud fi.Cloud, clusterName string) ([]*resources.Resource, error) { + gateways, err := DescribeEgressOnlyInternetGateways(cloud) + if err != nil { + return nil, err + } + + var resourceTrackers []*resources.Resource + + for _, o := range gateways { + resourceTracker := &resources.Resource{ + Name: FindName(o.Tags), + ID: aws.StringValue(o.EgressOnlyInternetGatewayId), + Type: "egress-only-internet-gateway", + Deleter: DeleteEgressOnlyInternetGateway, + Shared: HasSharedTag(ec2.ResourceTypeEgressOnlyInternetGateway+":"+aws.StringValue(o.EgressOnlyInternetGatewayId), o.Tags, clusterName), + } + + var blocks []string + for _, a := range o.Attachments { + if aws.StringValue(a.VpcId) != "" { + blocks = append(blocks, "vpc:"+aws.StringValue(a.VpcId)) + } + } + resourceTracker.Blocks = blocks + + resourceTrackers = append(resourceTrackers, resourceTracker) + } + + return resourceTrackers, nil +} + +func DescribeEgressOnlyInternetGateways(cloud fi.Cloud) ([]*ec2.EgressOnlyInternetGateway, error) { + c := cloud.(awsup.AWSCloud) + + klog.V(2).Infof("Listing EC2 EgressOnlyInternetGateways") + request := &ec2.DescribeEgressOnlyInternetGatewaysInput{ + Filters: BuildEC2Filters(cloud), + } + response, err := c.EC2().DescribeEgressOnlyInternetGateways(request) + if err != nil { + return nil, fmt.Errorf("error listing EgressOnlyInternetGateway: %v", err) + } + + var gateways []*ec2.EgressOnlyInternetGateway + gateways = append(gateways, response.EgressOnlyInternetGateways...) + + return gateways, nil +} + func DeleteAutoScalingGroup(cloud fi.Cloud, r *resources.Resource) error { c := cloud.(awsup.AWSCloud) diff --git a/upup/pkg/fi/cloudup/awstasks/BUILD.bazel b/upup/pkg/fi/cloudup/awstasks/BUILD.bazel index 01f1da69e1..0345b14b93 100644 --- a/upup/pkg/fi/cloudup/awstasks/BUILD.bazel +++ b/upup/pkg/fi/cloudup/awstasks/BUILD.bazel @@ -22,6 +22,8 @@ go_library( "dnszone_fitask.go", "ebsvolume.go", "ebsvolume_fitask.go", + "egressonlyinternetgateway.go", + "egressonlyinternetgateway_fitask.go", "elastic_ip.go", "elasticip_fitask.go", "eventbridgerule.go", @@ -120,6 +122,7 @@ go_test( srcs = [ "autoscalinggroup_test.go", "ebsvolume_test.go", + "egressonlyinternetgateway_test.go", "elastic_ip_test.go", "internetgateway_test.go", "launchtemplate_target_cloudformation_test.go", diff --git a/upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway.go b/upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway.go new file mode 100644 index 0000000000..e4e9d6a63a --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway.go @@ -0,0 +1,228 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awstasks + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/service/ec2" + "k8s.io/klog/v2" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + "k8s.io/kops/upup/pkg/fi/cloudup/cloudformation" + "k8s.io/kops/upup/pkg/fi/cloudup/terraform" + "k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter" +) + +// +kops:fitask +type EgressOnlyInternetGateway struct { + Name *string + Lifecycle fi.Lifecycle + + ID *string + VPC *VPC + // Shared is set if this is a shared EgressOnlyInternetGateway + Shared *bool + + // Tags is a map of aws tags that are added to the EgressOnlyInternetGateway + Tags map[string]string +} + +var _ fi.CompareWithID = &EgressOnlyInternetGateway{} + +func (e *EgressOnlyInternetGateway) CompareWithID() *string { + return e.ID +} + +func findEgressOnlyInternetGateway(cloud awsup.AWSCloud, request *ec2.DescribeEgressOnlyInternetGatewaysInput) (*ec2.EgressOnlyInternetGateway, error) { + response, err := cloud.EC2().DescribeEgressOnlyInternetGateways(request) + if err != nil { + return nil, fmt.Errorf("error listing EgressOnlyInternetGateways: %v", err) + } + if response == nil || len(response.EgressOnlyInternetGateways) == 0 { + return nil, nil + } + + if len(response.EgressOnlyInternetGateways) != 1 { + return nil, fmt.Errorf("found multiple EgressOnlyInternetGateways matching tags") + } + igw := response.EgressOnlyInternetGateways[0] + return igw, nil +} + +func (e *EgressOnlyInternetGateway) Find(c *fi.Context) (*EgressOnlyInternetGateway, error) { + cloud := c.Cloud.(awsup.AWSCloud) + + request := &ec2.DescribeEgressOnlyInternetGatewaysInput{} + + shared := fi.BoolValue(e.Shared) + if shared { + if fi.StringValue(e.VPC.ID) == "" { + return nil, fmt.Errorf("VPC ID is required when EgressOnlyInternetGateway is shared") + } + + request.Filters = []*ec2.Filter{awsup.NewEC2Filter("attachment.vpc-id", *e.VPC.ID)} + } else { + if e.ID != nil { + request.EgressOnlyInternetGatewayIds = []*string{e.ID} + } else { + request.Filters = cloud.BuildFilters(e.Name) + } + } + + eigw, err := findEgressOnlyInternetGateway(cloud, request) + if err != nil { + return nil, err + } + if eigw == nil { + return nil, nil + } + actual := &EgressOnlyInternetGateway{ + ID: eigw.EgressOnlyInternetGatewayId, + Name: findNameTag(eigw.Tags), + Tags: intersectTags(eigw.Tags, e.Tags), + } + + klog.V(2).Infof("found matching EgressOnlyInternetGateway %q", *actual.ID) + + for _, attachment := range eigw.Attachments { + actual.VPC = &VPC{ID: attachment.VpcId} + } + + // Prevent spurious comparison failures + actual.Shared = e.Shared + actual.Lifecycle = e.Lifecycle + if shared { + actual.Name = e.Name + } + if e.ID == nil { + e.ID = actual.ID + } + + // We don't set the tags for a shared EIGW + if fi.BoolValue(e.Shared) { + actual.Tags = e.Tags + } + + return actual, nil +} + +func (e *EgressOnlyInternetGateway) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +func (s *EgressOnlyInternetGateway) CheckChanges(a, e, changes *EgressOnlyInternetGateway) error { + if a != nil { + if changes.VPC != nil { + return fi.CannotChangeField("VPC") + } + } + + return nil +} + +func (_ *EgressOnlyInternetGateway) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EgressOnlyInternetGateway) error { + shared := fi.BoolValue(e.Shared) + if shared { + // Verify the EgressOnlyInternetGateway was found and matches our required settings + if a == nil { + return fmt.Errorf("EgressOnlyInternetGateway for shared VPC was not found") + } + + return nil + } + + if a == nil { + klog.V(2).Infof("Creating EgressOnlyInternetGateway") + + request := &ec2.CreateEgressOnlyInternetGatewayInput{ + VpcId: e.VPC.ID, + TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeEgressOnlyInternetGateway, e.Tags), + } + + response, err := t.Cloud.EC2().CreateEgressOnlyInternetGateway(request) + if err != nil { + return fmt.Errorf("error creating EgressOnlyInternetGateway: %v", err) + } + + e.ID = response.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId + return nil + } + + return t.UpdateTags(*e.ID, e.Tags) +} + +type terraformEgressOnlyInternetGateway struct { + VPCID *terraformWriter.Literal `json:"vpc_id" cty:"vpc_id"` + Tags map[string]string `json:"tags,omitempty" cty:"tags"` +} + +func (_ *EgressOnlyInternetGateway) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *EgressOnlyInternetGateway) error { + shared := fi.BoolValue(e.Shared) + if shared { + // Not terraform owned / managed + + // But ... attempt to discover the ID so TerraformLink works + if e.ID == nil { + request := &ec2.DescribeEgressOnlyInternetGatewaysInput{} + vpcID := fi.StringValue(e.VPC.ID) + if vpcID == "" { + return fmt.Errorf("VPC ID is required when EgressOnlyInternetGateway is shared") + } + request.Filters = []*ec2.Filter{awsup.NewEC2Filter("attachment.vpc-id", vpcID)} + igw, err := findEgressOnlyInternetGateway(t.Cloud.(awsup.AWSCloud), request) + if err != nil { + return err + } + if igw == nil { + klog.Warningf("Cannot find egress-only internet gateway for VPC %q", vpcID) + } else { + e.ID = igw.EgressOnlyInternetGatewayId + } + } + + return nil + } + + tf := &terraformEgressOnlyInternetGateway{ + VPCID: e.VPC.TerraformLink(), + Tags: e.Tags, + } + + return t.RenderResource("aws_egress_only_internet_gateway", *e.Name, tf) +} + +func (e *EgressOnlyInternetGateway) TerraformLink() *terraformWriter.Literal { + shared := fi.BoolValue(e.Shared) + if shared { + if e.ID == nil { + klog.Fatalf("ID must be set, if EgressOnlyInternetGateway is shared: %s", e) + } + + klog.V(4).Infof("reusing existing EgressOnlyInternetGateway with id %q", *e.ID) + return terraformWriter.LiteralFromStringValue(*e.ID) + } + + return terraformWriter.LiteralProperty("aws_egress_only_internet_gateway", *e.Name, "id") +} + +func (_ *EgressOnlyInternetGateway) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *EgressOnlyInternetGateway) error { + if changes != nil { + klog.Warning("Egress Only Internet Gateway is not supported by the cloudformation target") + } + return nil +} diff --git a/upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway_fitask.go b/upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway_fitask.go new file mode 100644 index 0000000000..f41ddd414d --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway_fitask.go @@ -0,0 +1,52 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by fitask. DO NOT EDIT. + +package awstasks + +import ( + "k8s.io/kops/upup/pkg/fi" +) + +// EgressOnlyInternetGateway + +var _ fi.HasLifecycle = &EgressOnlyInternetGateway{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *EgressOnlyInternetGateway) GetLifecycle() fi.Lifecycle { + return o.Lifecycle +} + +// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle +func (o *EgressOnlyInternetGateway) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = lifecycle +} + +var _ fi.HasName = &EgressOnlyInternetGateway{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *EgressOnlyInternetGateway) GetName() *string { + return o.Name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *EgressOnlyInternetGateway) String() string { + return fi.TaskAsString(o) +} diff --git a/upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway_test.go b/upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway_test.go new file mode 100644 index 0000000000..93101e65fa --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/egressonlyinternetgateway_test.go @@ -0,0 +1,144 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awstasks + +import ( + "reflect" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "k8s.io/kops/cloudmock/aws/mockec2" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" +) + +func TestSharedEgressOnlyInternetGatewayDoesNotRename(t *testing.T) { + cloud := awsup.BuildMockAWSCloud("us-east-1", "abc") + c := &mockec2.MockEC2{} + cloud.MockEC2 = c + + // Pre-create the vpc / subnet + vpc, err := c.CreateVpc(&ec2.CreateVpcInput{ + CidrBlock: aws.String("172.20.0.0/16"), + }) + if err != nil { + t.Fatalf("error creating test VPC: %v", err) + } + _, err = c.CreateTags(&ec2.CreateTagsInput{ + Resources: []*string{vpc.Vpc.VpcId}, + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("ExistingVPC"), + }, + }, + }) + if err != nil { + t.Fatalf("error tagging test vpc: %v", err) + } + + internetGateway, err := c.CreateEgressOnlyInternetGateway(&ec2.CreateEgressOnlyInternetGatewayInput{ + VpcId: vpc.Vpc.VpcId, + TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeEgressOnlyInternetGateway, map[string]string{ + "Name": "ExistingInternetGateway", + }), + }) + if err != nil { + t.Fatalf("error creating test eigw: %v", err) + } + + // We define a function so we can rebuild the tasks, because we modify in-place when running + buildTasks := func() map[string]fi.Task { + vpc1 := &VPC{ + Name: s("vpc1"), + Lifecycle: fi.LifecycleSync, + CIDR: s("172.20.0.0/16"), + Tags: map[string]string{"kubernetes.io/cluster/cluster.example.com": "shared"}, + Shared: fi.Bool(true), + ID: vpc.Vpc.VpcId, + } + eigw1 := &EgressOnlyInternetGateway{ + Name: s("eigw1"), + Lifecycle: fi.LifecycleSync, + VPC: vpc1, + Shared: fi.Bool(true), + ID: internetGateway.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId, + Tags: make(map[string]string), + } + + return map[string]fi.Task{ + "eigw1": eigw1, + "vpc1": vpc1, + } + } + + { + allTasks := buildTasks() + eigw1 := allTasks["eigw1"].(*EgressOnlyInternetGateway) + + target := &awsup.AWSAPITarget{ + Cloud: cloud, + } + + context, err := fi.NewContext(target, nil, cloud, nil, nil, nil, true, allTasks) + if err != nil { + t.Fatalf("error building context: %v", err) + } + defer context.Close() + + if err := context.RunTasks(testRunTasksOptions); err != nil { + t.Fatalf("unexpected error during Run: %v", err) + } + + if fi.StringValue(eigw1.ID) == "" { + t.Fatalf("ID not set after create") + } + + if len(c.EgressOnlyInternetGatewayIds()) != 1 { + t.Fatalf("Expected exactly one EgressOnlyInternetGateway; found %v", c.EgressOnlyInternetGatewayIds()) + } + + actual := c.FindEgressOnlyInternetGateway(*internetGateway.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId) + if actual == nil { + t.Fatalf("EgressOnlyInternetGateway created but then not found") + } + expected := &ec2.EgressOnlyInternetGateway{ + EgressOnlyInternetGatewayId: aws.String("eigw-1"), + Tags: buildTags(map[string]string{ + "Name": "ExistingInternetGateway", + }), + Attachments: []*ec2.InternetGatewayAttachment{ + { + VpcId: vpc.Vpc.VpcId, + }, + }, + } + + mockec2.SortTags(expected.Tags) + mockec2.SortTags(actual.Tags) + + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("Unexpected EgressOnlyInternetGateway: expected=%v actual=%v", expected, actual) + } + } + + { + allTasks := buildTasks() + checkNoChanges(t, cloud, allTasks) + } +} diff --git a/upup/pkg/fi/cloudup/awstasks/route.go b/upup/pkg/fi/cloudup/awstasks/route.go index 0e4ad99ec7..a0b2723294 100644 --- a/upup/pkg/fi/cloudup/awstasks/route.go +++ b/upup/pkg/fi/cloudup/awstasks/route.go @@ -41,9 +41,10 @@ type Route struct { // Exactly one of the below fields // MUST be provided. - InternetGateway *InternetGateway - NatGateway *NatGateway - TransitGatewayID *string + EgressOnlyInternetGateway *EgressOnlyInternetGateway + InternetGateway *InternetGateway + NatGateway *NatGateway + TransitGatewayID *string } func (e *Route) Find(c *fi.Context) (*Route, error) { @@ -84,15 +85,18 @@ func (e *Route) Find(c *fi.Context) (*Route, error) { CIDR: r.DestinationCidrBlock, IPv6CIDR: r.DestinationIpv6CidrBlock, } + if r.EgressOnlyInternetGatewayId != nil { + actual.EgressOnlyInternetGateway = &EgressOnlyInternetGateway{ID: r.EgressOnlyInternetGatewayId} + } if r.GatewayId != nil { actual.InternetGateway = &InternetGateway{ID: r.GatewayId} } - if r.NatGatewayId != nil { - actual.NatGateway = &NatGateway{ID: r.NatGatewayId} - } if r.InstanceId != nil { actual.Instance = &Instance{ID: r.InstanceId} } + if r.NatGatewayId != nil { + actual.NatGateway = &NatGateway{ID: r.NatGatewayId} + } if r.TransitGatewayId != nil { actual.TransitGatewayID = r.TransitGatewayId } @@ -130,9 +134,15 @@ func (s *Route) CheckChanges(a, e, changes *Route) error { return fi.RequiredField("CIDR/IPv6CIDR") } if e.CIDR != nil && e.IPv6CIDR != nil { - return fmt.Errorf("cannot set more than 1 CIDR or IPv6CIDR") + return fmt.Errorf("cannot set more than one CIDR or IPv6CIDR") } targetCount := 0 + if e.EgressOnlyInternetGateway != nil { + targetCount++ + if e.CIDR != nil { + return fmt.Errorf("cannot route IPv4 to an EgressOnlyInternetGateway") + } + } if e.InternetGateway != nil { targetCount++ } @@ -146,10 +156,10 @@ func (s *Route) CheckChanges(a, e, changes *Route) error { targetCount++ } if targetCount == 0 { - return fmt.Errorf("InternetGateway, Instance, NatGateway, or TransitGateway is required") + return fmt.Errorf("EgressOnlyInternetGateway, InternetGateway, Instance, NatGateway, or TransitGateway is required") } if targetCount != 1 { - return fmt.Errorf("cannot set more than 1 InternetGateway, Instance, NatGateway, or TransitGateway") + return fmt.Errorf("cannot set more than one EgressOnlyInternetGateway, InternetGateway, Instance, NatGateway, or TransitGateway") } } @@ -179,8 +189,10 @@ func (_ *Route) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Route) error { klog.Fatal("both CIDR and IPv6CIDR were unexpectedly nil") } - if e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil { + if e.EgressOnlyInternetGateway == nil && e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil { return fmt.Errorf("missing target for route") + } else if e.EgressOnlyInternetGateway != nil { + request.EgressOnlyInternetGatewayId = checkNotNil(e.EgressOnlyInternetGateway.ID) } else if e.InternetGateway != nil { request.GatewayId = checkNotNil(e.InternetGateway.ID) } else if e.NatGateway != nil { @@ -259,13 +271,14 @@ func checkNotNil(s *string) *string { } type terraformRoute struct { - RouteTableID *terraformWriter.Literal `json:"route_table_id" cty:"route_table_id"` - CIDR *string `json:"destination_cidr_block,omitempty" cty:"destination_cidr_block"` - IPv6CIDR *string `json:"destination_ipv6_cidr_block,omitempty" cty:"destination_ipv6_cidr_block"` - InternetGatewayID *terraformWriter.Literal `json:"gateway_id,omitempty" cty:"gateway_id"` - NATGatewayID *terraformWriter.Literal `json:"nat_gateway_id,omitempty" cty:"nat_gateway_id"` - TransitGatewayID *string `json:"transit_gateway_id,omitempty" cty:"transit_gateway_id"` - InstanceID *terraformWriter.Literal `json:"instance_id,omitempty" cty:"instance_id"` + RouteTableID *terraformWriter.Literal `json:"route_table_id" cty:"route_table_id"` + CIDR *string `json:"destination_cidr_block,omitempty" cty:"destination_cidr_block"` + IPv6CIDR *string `json:"destination_ipv6_cidr_block,omitempty" cty:"destination_ipv6_cidr_block"` + EgressOnlyInternetGatewayID *terraformWriter.Literal `json:"egress_onlygateway_id,omitempty" cty:"egress_only_gateway_id"` + InternetGatewayID *terraformWriter.Literal `json:"gateway_id,omitempty" cty:"gateway_id"` + NATGatewayID *terraformWriter.Literal `json:"nat_gateway_id,omitempty" cty:"nat_gateway_id"` + TransitGatewayID *string `json:"transit_gateway_id,omitempty" cty:"transit_gateway_id"` + InstanceID *terraformWriter.Literal `json:"instance_id,omitempty" cty:"instance_id"` } func (_ *Route) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Route) error { @@ -275,8 +288,10 @@ func (_ *Route) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Rou IPv6CIDR: e.IPv6CIDR, } - if e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil { + if e.EgressOnlyInternetGateway == nil && e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil { return fmt.Errorf("missing target for route") + } else if e.EgressOnlyInternetGateway != nil { + tf.EgressOnlyInternetGatewayID = e.EgressOnlyInternetGateway.TerraformLink() } else if e.InternetGateway != nil { tf.InternetGatewayID = e.InternetGateway.TerraformLink() } else if e.NatGateway != nil { diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index 5b712c79f4..d24bf6b7ee 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -984,19 +984,6 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S cluster.Spec.Subnets[i].Type = api.SubnetTypePublic } - if opt.IPv6 { - cluster.Spec.NonMasqueradeCIDR = "::/0" - cluster.Spec.ExternalCloudControllerManager = &api.CloudControllerManagerConfig{} - if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS { - klog.Warningf("IPv6 support is EXPERIMENTAL and can be changed or removed at any time in the future!!!") - for i := range cluster.Spec.Subnets { - cluster.Spec.Subnets[i].IPv6CIDR = fmt.Sprintf("/64#%x", i) - } - } else { - klog.Errorf("IPv6 support is available only on AWS") - } - } - case api.TopologyPrivate: if cluster.Spec.Networking.Kubenet != nil { return nil, fmt.Errorf("invalid networking option %s. Kubenet does not support private topology", opt.Networking) @@ -1085,6 +1072,19 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S return nil, fmt.Errorf("invalid topology %s", opt.Topology) } + if opt.IPv6 { + cluster.Spec.NonMasqueradeCIDR = "::/0" + cluster.Spec.ExternalCloudControllerManager = &api.CloudControllerManagerConfig{} + if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS { + klog.Warningf("IPv6 support is EXPERIMENTAL and can be changed or removed at any time in the future!!!") + for i := range cluster.Spec.Subnets { + cluster.Spec.Subnets[i].IPv6CIDR = fmt.Sprintf("/64#%x", i) + } + } else { + klog.Errorf("IPv6 support is available only on AWS") + } + } + cluster.Spec.Topology.DNS = &api.DNSSpec{} switch strings.ToLower(opt.DNSType) { case "public", "":