From 98c1109cc65f8328337e84008bfc931faec60926 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Sat, 23 Apr 2022 14:47:05 -0400 Subject: [PATCH] gce: Add IPv6 support to subnet/instances We need to specify StackType & IPv6AccessType --- pkg/model/gcemodel/autoscalinggroup.go | 14 ++++ pkg/model/gcemodel/network.go | 14 ++++ upup/pkg/fi/cloudup/gcetasks/instance.go | 31 +++++---- .../fi/cloudup/gcetasks/instancetemplate.go | 15 ++++- upup/pkg/fi/cloudup/gcetasks/subnet.go | 66 ++++++++++++++++--- 5 files changed, 118 insertions(+), 22 deletions(-) diff --git a/pkg/model/gcemodel/autoscalinggroup.go b/pkg/model/gcemodel/autoscalinggroup.go index 93ec8ce047..1c7ee2bd94 100644 --- a/pkg/model/gcemodel/autoscalinggroup.go +++ b/pkg/model/gcemodel/autoscalinggroup.go @@ -119,6 +119,20 @@ func (b *AutoscalingGroupModelBuilder) buildInstanceTemplate(c *fi.CloudupModelB t.Metadata["kube-env"] = fi.NewStringResource("AUTOSCALER_ENV_VARS: " + autoscalerEnvVars) } + stackType := "IPV4_ONLY" + if b.IsIPv6Only() { + // The subnets are dual-mode; IPV6_ONLY is not yet supported. + // This means that VMs will get an IPv4 and a /96 IPv6. + // However, pods will still be IPv6 only. + stackType = "IPV4_IPV6" + + // // Ipv6AccessType must be set when enabling IPv6. + // // EXTERNAL is currently the only supported value + // ipv6AccessType := "EXTERNAL" + // t.Ipv6AccessType = &ipv6AccessType + } + t.StackType = &stackType + nodeRole, err := iam.BuildNodeRoleSubject(ig.Spec.Role, false) if err != nil { return nil, err diff --git a/pkg/model/gcemodel/network.go b/pkg/model/gcemodel/network.go index 57b6d3632c..452219b62f 100644 --- a/pkg/model/gcemodel/network.go +++ b/pkg/model/gcemodel/network.go @@ -73,6 +73,20 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error { t.CIDR = s(subnet.CIDR) } + stackType := "IPV4_ONLY" + if b.IsIPv6Only() { + // The subnets are dual-mode; IPV6_ONLY is not yet supported. + // This means that VMs will get an IPv4 and a /96 IPv6. + // However, pods will still be IPv6 only. + stackType = "IPV4_IPV6" + + // Ipv6AccessType must be set when enabling IPv6. + // EXTERNAL is currently the only supported value + ipv6AccessType := "EXTERNAL" + t.Ipv6AccessType = &ipv6AccessType + } + t.StackType = &stackType + t.SecondaryIpRanges = make(map[string]string) if gce.UsesIPAliases(b.Cluster) { // The primary CIDR is used by the nodes, diff --git a/upup/pkg/fi/cloudup/gcetasks/instance.go b/upup/pkg/fi/cloudup/gcetasks/instance.go index d42c9f7aaf..9b43e8163c 100644 --- a/upup/pkg/fi/cloudup/gcetasks/instance.go +++ b/upup/pkg/fi/cloudup/gcetasks/instance.go @@ -46,6 +46,7 @@ type Instance struct { CanIPForward *bool IPAddress *Address Subnet *Subnet + StackType *string Scopes []string @@ -79,13 +80,13 @@ func (e *Instance) Find(c *fi.CloudupContext) (*Instance, error) { actual.Zone = fi.PtrTo(lastComponent(r.Zone)) actual.MachineType = fi.PtrTo(lastComponent(r.MachineType)) actual.CanIPForward = &r.CanIpForward - if r.Scheduling != nil { actual.Preemptible = &r.Scheduling.Preemptible } if len(r.NetworkInterfaces) != 0 { ni := r.NetworkInterfaces[0] actual.Network = &Network{Name: fi.PtrTo(lastComponent(ni.Network))} + actual.StackType = &ni.StackType if len(ni.AccessConfigs) != 0 { ac := ni.AccessConfigs[0] if ac.NatIP != "" { @@ -233,24 +234,30 @@ func (e *Instance) mapToGCE(project string, ipAddressResolver func(*Address) (*s } var networkInterfaces []*compute.NetworkInterface - if e.IPAddress != nil { - addr, err := ipAddressResolver(e.IPAddress) - if err != nil { - return nil, fmt.Errorf("unable to resolve IP for instance: %v", err) - } - if addr == nil { - return nil, fmt.Errorf("instance IP address has not yet been created") - } + { networkInterface := &compute.NetworkInterface{ AccessConfigs: []*compute.AccessConfig{{ - NatIP: *addr, - Type: "ONE_TO_ONE_NAT", + Type: "ONE_TO_ONE_NAT", }}, Network: e.Network.URL(project), } + + if e.IPAddress != nil { + addr, err := ipAddressResolver(e.IPAddress) + if err != nil { + return nil, fmt.Errorf("unable to resolve IP for instance: %v", err) + } + if addr == nil { + return nil, fmt.Errorf("instance IP address has not yet been created") + } + networkInterface.AccessConfigs[0].NatIP = *addr + } if e.Subnet != nil { networkInterface.Subnetwork = *e.Subnet.Name } + if e.StackType != nil { + networkInterface.StackType = *e.StackType + } networkInterfaces = append(networkInterfaces, networkInterface) } @@ -466,7 +473,7 @@ func (_ *Instance) RenderTerraform(t *terraform.TerraformTarget, a, e, changes * tf.Disks = append(tf.Disks, tfd) } - tf.NetworkInterfaces = addNetworks(e.Network, e.Subnet, i.NetworkInterfaces) + tf.NetworkInterfaces = addNetworks(e.StackType, e.Network, e.Subnet, i.NetworkInterfaces) metadata, err := addMetadata(t, i.Name, i.Metadata) if err != nil { diff --git a/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go b/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go index a57b40ee64..bc5d5d039c 100644 --- a/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go +++ b/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go @@ -74,6 +74,9 @@ type InstanceTemplate struct { // HasExternalIP is set to true when an external IP is allocated to an instance. HasExternalIP *bool + // StackType indicates the address families supported (IPV4_IPV6 or IPV4_ONLY) + StackType *string + // ID is the actual name ID *string @@ -138,6 +141,9 @@ func (e *InstanceTemplate) Find(c *fi.CloudupContext) (*InstanceTemplate, error) if len(p.NetworkInterfaces) != 0 { ni := p.NetworkInterfaces[0] actual.Network = &Network{Name: fi.PtrTo(lastComponent(ni.Network))} + if ni.StackType != "" { + actual.StackType = &ni.StackType + } if len(ni.AliasIpRanges) != 0 { actual.AliasIPRanges = make(map[string]string) @@ -312,6 +318,9 @@ func (e *InstanceTemplate) mapToGCE(project string, region string) (*compute.Ins }, } } + if e.StackType != nil { + ni.StackType = fi.ValueOf(e.StackType) + } if e.Subnet != nil { ni.Subnetwork = e.Subnet.URL(networkProject, region) @@ -526,6 +535,7 @@ type terraformNetworkInterface struct { Network *terraformWriter.Literal `cty:"network"` Subnetwork *terraformWriter.Literal `cty:"subnetwork"` AccessConfig []*terraformAccessConfig `cty:"access_config"` + StackType *string `cty:"stack_type"` } type terraformAccessConfig struct { @@ -537,10 +547,11 @@ type terraformGuestAccelerator struct { Count int64 `cty:"count"` } -func addNetworks(network *Network, subnet *Subnet, networkInterfaces []*compute.NetworkInterface) []*terraformNetworkInterface { +func addNetworks(stackType *string, network *Network, subnet *Subnet, networkInterfaces []*compute.NetworkInterface) []*terraformNetworkInterface { ni := make([]*terraformNetworkInterface, 0) for _, g := range networkInterfaces { tf := &terraformNetworkInterface{} + tf.StackType = stackType if network != nil { tf.Network = network.TerraformLink() } @@ -639,7 +650,7 @@ func (_ *InstanceTemplate) RenderTerraform(t *terraform.TerraformTarget, a, e, c tf.Disks = append(tf.Disks, tfd) } - tf.NetworkInterfaces = addNetworks(e.Network, e.Subnet, i.Properties.NetworkInterfaces) + tf.NetworkInterfaces = addNetworks(e.StackType, e.Network, e.Subnet, i.Properties.NetworkInterfaces) metadata, err := addMetadata(t, name, i.Properties.Metadata) if err != nil { diff --git a/upup/pkg/fi/cloudup/gcetasks/subnet.go b/upup/pkg/fi/cloudup/gcetasks/subnet.go index 38f854b4fb..2508d8d809 100644 --- a/upup/pkg/fi/cloudup/gcetasks/subnet.go +++ b/upup/pkg/fi/cloudup/gcetasks/subnet.go @@ -37,6 +37,12 @@ type Subnet struct { Region *string CIDR *string + // StackType indicates the address families supported (IPV4_IPV6 or IPV4_ONLY) + StackType *string + + // Ipv6AccessType indicates whether the IPv6 addresses are accessible externally + Ipv6AccessType *string + SecondaryIpRanges map[string]string Shared *bool @@ -69,6 +75,8 @@ func (e *Subnet) Find(c *fi.CloudupContext) (*Subnet, error) { actual.Network = &Network{Name: fi.PtrTo(lastComponent(s.Network))} actual.Region = fi.PtrTo(lastComponent(s.Region)) actual.CIDR = &s.IpCidrRange + actual.StackType = &s.StackType + actual.Ipv6AccessType = &s.Ipv6AccessType shared := fi.ValueOf(e.Shared) { @@ -117,9 +125,11 @@ func (_ *Subnet) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Subnet) error { klog.V(2).Infof("Creating Subnet with CIDR: %q", fi.ValueOf(e.CIDR)) subnet := &compute.Subnetwork{ - IpCidrRange: fi.ValueOf(e.CIDR), - Name: *e.Name, - Network: e.Network.URL(project), + IpCidrRange: fi.ValueOf(e.CIDR), + Name: *e.Name, + Network: e.Network.URL(project), + StackType: fi.ValueOf(e.StackType), + Ipv6AccessType: fi.ValueOf(e.Ipv6AccessType), } for k, v := range e.SecondaryIpRanges { @@ -152,6 +162,15 @@ func (_ *Subnet) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Subnet) error { changes.SecondaryIpRanges = nil } + if changes.StackType != nil { + if err := updateStackTypeAndIPv6AccessType(cloud, e); err != nil { + return err + } + + changes.StackType = nil + changes.Ipv6AccessType = nil + } + empty := &Subnet{} if !reflect.DeepEqual(empty, changes) { return fmt.Errorf("cannot apply changes to Subnet: %v", changes) @@ -217,11 +236,37 @@ func updateSecondaryRanges(cloud gce.GCECloud, op string, e *Subnet) error { } } - _, err = cloud.Compute().Subnetworks().Patch(cloud.Project(), cloud.Region(), subnet.Name, subnet) + patchOp, err := cloud.Compute().Subnetworks().Patch(cloud.Project(), cloud.Region(), subnet.Name, subnet) if err != nil { return fmt.Errorf("error patching Subnet: %w", err) } + if err := cloud.WaitForOp(patchOp); err != nil { + return fmt.Errorf("error waiting for Subnet patch to complete: %w", err) + } + + return nil +} + +func updateStackTypeAndIPv6AccessType(cloud gce.GCECloud, e *Subnet) error { + // We need to refetch to patch it + subnet, err := cloud.Compute().Subnetworks().Get(cloud.Project(), cloud.Region(), *e.Name) + if err != nil { + return fmt.Errorf("error fetching subnet for patch: %w", err) + } + + subnet.StackType = fi.ValueOf(e.StackType) + subnet.Ipv6AccessType = fi.ValueOf(e.Ipv6AccessType) + + patchOp, err := cloud.Compute().Subnetworks().Patch(cloud.Project(), cloud.Region(), subnet.Name, subnet) + if err != nil { + return fmt.Errorf("error patching Subnet: %w", err) + } + + if err := cloud.WaitForOp(patchOp); err != nil { + return fmt.Errorf("error waiting for Subnet patch to complete: %w", err) + } + return nil } @@ -244,6 +289,9 @@ type terraformSubnet struct { // SecondaryIPRange defines additional IP ranges SecondaryIPRange []terraformSubnetRange `cty:"secondary_ip_range"` + + StackType *string `cty:"stack_type"` + Ipv6AccessType *string `cty:"ipv6_access_type"` } type terraformSubnetRange struct { @@ -259,10 +307,12 @@ func (_ *Subnet) RenderSubnet(t *terraform.TerraformTarget, a, e, changes *Subne } tf := &terraformSubnet{ - Name: e.Name, - Network: e.Network.TerraformLink(), - Region: e.Region, - CIDR: e.CIDR, + Name: e.Name, + Network: e.Network.TerraformLink(), + Region: e.Region, + CIDR: e.CIDR, + StackType: e.StackType, + Ipv6AccessType: e.Ipv6AccessType, } for k, v := range e.SecondaryIpRanges {