Merge pull request #13318 from guillomep/allow_to_add_additional_routes

Possibility to add additional routes in route tables of subnets

docs/cluster_spec.md

@@ -351,6 +351,25 @@ spec:
     zone: us-east-1a
 ```
 
+### additionalRoutes
+
+{{ kops_feature_table(kops_added_default='1.24') }}
+
+Add routes in the route tables of the subnet. Targets of routes can be an instance, a peering connection, a NAT gateway, a transit gateway, an internet gateway or an egress-only internet gateway.
+Currently, only AWS is supported.
+
+```yaml
+spec:
+  subnets:
+  - cidr: 10.20.64.0/21
+    name: us-east-1a
+    type: Private
+    zone: us-east-1a
+    additionalRoutes:
+    - cidr: 10.21.0.0/16
+      target: vpc-abcdef
+```
+
 ## kubeAPIServer
 
 This block contains configuration for the `kube-apiserver`.
@@ -1459,4 +1478,4 @@ spec:
             ]
 ```
 
-To configure Pods to assume the given IAM roles, enable the [Pod Identity Webhook](/addons/#pod-identity-webhook). Without this webhook, you need to modify your Pod specs yourself for your Pod to assume the defined roles.
+To configure Pods to assume the given IAM roles, enable the [Pod Identity Webhook](/addons/#pod-identity-webhook). Without this webhook, you need to modify your Pod specs yourself for your Pod to assume the defined roles.

k8s/crds/kops.k8s.io_clusters.yaml

@@ -5378,6 +5378,19 @@ spec:
                 description: Configuration of subnets we are targeting
                 items:
                   properties:
+                    additionalRoutes:
+                      description: AdditionalRoutes to attach to the subnet's route
+                        table
+                      items:
+                        properties:
+                          cidr:
+                            description: CIDR destination of the route
+                            type: string
+                          target:
+                            description: Target of the route
+                            type: string
+                        type: object
+                      type: array
                     cidr:
                       description: CIDR is the IPv4 CIDR block assigned to the subnet.
                       type: string

pkg/apis/kops/cluster.go

@@ -725,6 +725,15 @@ type ClusterSubnetSpec struct {
 	Type SubnetType `json:"type,omitempty"`
 	// PublicIP to attach to NatGateway
 	PublicIP string `json:"publicIP,omitempty"`
+	// AdditionalRoutes to attach to the subnet's route table
+	AdditionalRoutes []RouteSpec `json:"additionalRoutes,omitempty"`
+}
+
+type RouteSpec struct {
+	// CIDR destination of the route
+	CIDR string `json:"cidr,omitempty"`
+	// Target of the route
+	Target string `json:"target,omitempty"`
 }
 
 type EgressProxySpec struct {

pkg/apis/kops/v1alpha2/cluster.go

@@ -689,6 +689,16 @@ type ClusterSubnetSpec struct {
 	Type SubnetType `json:"type,omitempty"`
 	// PublicIP to attach to NatGateway
 	PublicIP string `json:"publicIP,omitempty"`
+
+	// AdditionalRoutes to attach to the subnet's route table
+	AdditionalRoutes []RouteSpec `json:"additionalRoutes,omitempty"`
+}
+
+type RouteSpec struct {
+	// CIDR destination of the route
+	CIDR string `json:"cidr,omitempty"`
+	// Target of the route
+	Target string `json:"target,omitempty"`
 }
 
 type EgressProxySpec struct {

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1024,6 +1024,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*RouteSpec)(nil), (*kops.RouteSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_RouteSpec_To_kops_RouteSpec(a.(*RouteSpec), b.(*kops.RouteSpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.RouteSpec)(nil), (*RouteSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_RouteSpec_To_v1alpha2_RouteSpec(a.(*kops.RouteSpec), b.(*RouteSpec), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*SSHCredential)(nil), (*kops.SSHCredential)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_SSHCredential_To_kops_SSHCredential(a.(*SSHCredential), b.(*kops.SSHCredential), scope)
 	}); err != nil {
@@ -3223,6 +3233,17 @@ func autoConvert_v1alpha2_ClusterSubnetSpec_To_kops_ClusterSubnetSpec(in *Cluste
 	out.Egress = in.Egress
 	out.Type = kops.SubnetType(in.Type)
 	out.PublicIP = in.PublicIP
+	if in.AdditionalRoutes != nil {
+		in, out := &in.AdditionalRoutes, &out.AdditionalRoutes
+		*out = make([]kops.RouteSpec, len(*in))
+		for i := range *in {
+			if err := Convert_v1alpha2_RouteSpec_To_kops_RouteSpec(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.AdditionalRoutes = nil
+	}
 	return nil
 }
 
@@ -3241,6 +3262,17 @@ func autoConvert_kops_ClusterSubnetSpec_To_v1alpha2_ClusterSubnetSpec(in *kops.C
 	out.Egress = in.Egress
 	out.Type = SubnetType(in.Type)
 	out.PublicIP = in.PublicIP
+	if in.AdditionalRoutes != nil {
+		in, out := &in.AdditionalRoutes, &out.AdditionalRoutes
+		*out = make([]RouteSpec, len(*in))
+		for i := range *in {
+			if err := Convert_kops_RouteSpec_To_v1alpha2_RouteSpec(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.AdditionalRoutes = nil
+	}
 	return nil
 }
 
@@ -6824,6 +6856,28 @@ func Convert_kops_RomanaNetworkingSpec_To_v1alpha2_RomanaNetworkingSpec(in *kops
 	return autoConvert_kops_RomanaNetworkingSpec_To_v1alpha2_RomanaNetworkingSpec(in, out, s)
 }
 
+func autoConvert_v1alpha2_RouteSpec_To_kops_RouteSpec(in *RouteSpec, out *kops.RouteSpec, s conversion.Scope) error {
+	out.CIDR = in.CIDR
+	out.Target = in.Target
+	return nil
+}
+
+// Convert_v1alpha2_RouteSpec_To_kops_RouteSpec is an autogenerated conversion function.
+func Convert_v1alpha2_RouteSpec_To_kops_RouteSpec(in *RouteSpec, out *kops.RouteSpec, s conversion.Scope) error {
+	return autoConvert_v1alpha2_RouteSpec_To_kops_RouteSpec(in, out, s)
+}
+
+func autoConvert_kops_RouteSpec_To_v1alpha2_RouteSpec(in *kops.RouteSpec, out *RouteSpec, s conversion.Scope) error {
+	out.CIDR = in.CIDR
+	out.Target = in.Target
+	return nil
+}
+
+// Convert_kops_RouteSpec_To_v1alpha2_RouteSpec is an autogenerated conversion function.
+func Convert_kops_RouteSpec_To_v1alpha2_RouteSpec(in *kops.RouteSpec, out *RouteSpec, s conversion.Scope) error {
+	return autoConvert_kops_RouteSpec_To_v1alpha2_RouteSpec(in, out, s)
+}
+
 func autoConvert_v1alpha2_SSHCredential_To_kops_SSHCredential(in *SSHCredential, out *kops.SSHCredential, s conversion.Scope) error {
 	out.ObjectMeta = in.ObjectMeta
 	if err := Convert_v1alpha2_SSHCredentialSpec_To_kops_SSHCredentialSpec(&in.Spec, &out.Spec, s); err != nil {

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -973,7 +973,9 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 	if in.Subnets != nil {
 		in, out := &in.Subnets, &out.Subnets
 		*out = make([]ClusterSubnetSpec, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	if in.AdditionalNetworkCIDRs != nil {
 		in, out := &in.AdditionalNetworkCIDRs, &out.AdditionalNetworkCIDRs
@@ -1289,6 +1291,11 @@ func (in *ClusterSpec) DeepCopy() *ClusterSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClusterSubnetSpec) DeepCopyInto(out *ClusterSubnetSpec) {
 	*out = *in
+	if in.AdditionalRoutes != nil {
+		in, out := &in.AdditionalRoutes, &out.AdditionalRoutes
+		*out = make([]RouteSpec, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -4867,6 +4874,22 @@ func (in *RomanaNetworkingSpec) DeepCopy() *RomanaNetworkingSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RouteSpec) DeepCopyInto(out *RouteSpec) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteSpec.
+func (in *RouteSpec) DeepCopy() *RouteSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(RouteSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SSHCredential) DeepCopyInto(out *SSHCredential) {
 	*out = *in

pkg/apis/kops/v1alpha3/cluster.go

@@ -667,6 +667,16 @@ type ClusterSubnetSpec struct {
 	Type SubnetType `json:"type,omitempty"`
 	// PublicIP to attach to NatGateway
 	PublicIP string `json:"publicIP,omitempty"`
+
+	// AdditionalRoutes to attach to the subnet's route table
+	AdditionalRoutes []RouteSpec `json:"additionalRoutes,omitempty"`
+}
+
+type RouteSpec struct {
+	// CIDR destination of the route
+	CIDR string `json:"cidr,omitempty"`
+	// Target of the route
+	Target string `json:"target,omitempty"`
 }
 
 type EgressProxySpec struct {

pkg/apis/kops/v1alpha3/zz_generated.conversion.go

@@ -1074,6 +1074,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*RouteSpec)(nil), (*kops.RouteSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha3_RouteSpec_To_kops_RouteSpec(a.(*RouteSpec), b.(*kops.RouteSpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.RouteSpec)(nil), (*RouteSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_RouteSpec_To_v1alpha3_RouteSpec(a.(*kops.RouteSpec), b.(*RouteSpec), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*SSHCredential)(nil), (*kops.SSHCredential)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha3_SSHCredential_To_kops_SSHCredential(a.(*SSHCredential), b.(*kops.SSHCredential), scope)
 	}); err != nil {
@@ -3315,6 +3325,17 @@ func autoConvert_v1alpha3_ClusterSubnetSpec_To_kops_ClusterSubnetSpec(in *Cluste
 	out.Egress = in.Egress
 	out.Type = kops.SubnetType(in.Type)
 	out.PublicIP = in.PublicIP
+	if in.AdditionalRoutes != nil {
+		in, out := &in.AdditionalRoutes, &out.AdditionalRoutes
+		*out = make([]kops.RouteSpec, len(*in))
+		for i := range *in {
+			if err := Convert_v1alpha3_RouteSpec_To_kops_RouteSpec(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.AdditionalRoutes = nil
+	}
 	return nil
 }
 
@@ -3333,6 +3354,17 @@ func autoConvert_kops_ClusterSubnetSpec_To_v1alpha3_ClusterSubnetSpec(in *kops.C
 	out.Egress = in.Egress
 	out.Type = SubnetType(in.Type)
 	out.PublicIP = in.PublicIP
+	if in.AdditionalRoutes != nil {
+		in, out := &in.AdditionalRoutes, &out.AdditionalRoutes
+		*out = make([]RouteSpec, len(*in))
+		for i := range *in {
+			if err := Convert_kops_RouteSpec_To_v1alpha3_RouteSpec(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.AdditionalRoutes = nil
+	}
 	return nil
 }
 
@@ -6813,6 +6845,28 @@ func Convert_kops_RollingUpdate_To_v1alpha3_RollingUpdate(in *kops.RollingUpdate
 	return autoConvert_kops_RollingUpdate_To_v1alpha3_RollingUpdate(in, out, s)
 }
 
+func autoConvert_v1alpha3_RouteSpec_To_kops_RouteSpec(in *RouteSpec, out *kops.RouteSpec, s conversion.Scope) error {
+	out.CIDR = in.CIDR
+	out.Target = in.Target
+	return nil
+}
+
+// Convert_v1alpha3_RouteSpec_To_kops_RouteSpec is an autogenerated conversion function.
+func Convert_v1alpha3_RouteSpec_To_kops_RouteSpec(in *RouteSpec, out *kops.RouteSpec, s conversion.Scope) error {
+	return autoConvert_v1alpha3_RouteSpec_To_kops_RouteSpec(in, out, s)
+}
+
+func autoConvert_kops_RouteSpec_To_v1alpha3_RouteSpec(in *kops.RouteSpec, out *RouteSpec, s conversion.Scope) error {
+	out.CIDR = in.CIDR
+	out.Target = in.Target
+	return nil
+}
+
+// Convert_kops_RouteSpec_To_v1alpha3_RouteSpec is an autogenerated conversion function.
+func Convert_kops_RouteSpec_To_v1alpha3_RouteSpec(in *kops.RouteSpec, out *RouteSpec, s conversion.Scope) error {
+	return autoConvert_kops_RouteSpec_To_v1alpha3_RouteSpec(in, out, s)
+}
+
 func autoConvert_v1alpha3_SSHCredential_To_kops_SSHCredential(in *SSHCredential, out *kops.SSHCredential, s conversion.Scope) error {
 	out.ObjectMeta = in.ObjectMeta
 	if err := Convert_v1alpha3_SSHCredentialSpec_To_kops_SSHCredentialSpec(&in.Spec, &out.Spec, s); err != nil {

pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go

@@ -957,7 +957,9 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 	if in.Subnets != nil {
 		in, out := &in.Subnets, &out.Subnets
 		*out = make([]ClusterSubnetSpec, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	if in.AdditionalNetworkCIDRs != nil {
 		in, out := &in.AdditionalNetworkCIDRs, &out.AdditionalNetworkCIDRs
@@ -1273,6 +1275,11 @@ func (in *ClusterSpec) DeepCopy() *ClusterSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClusterSubnetSpec) DeepCopyInto(out *ClusterSubnetSpec) {
 	*out = *in
+	if in.AdditionalRoutes != nil {
+		in, out := &in.AdditionalRoutes, &out.AdditionalRoutes
+		*out = make([]RouteSpec, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -4798,6 +4805,22 @@ func (in *RollingUpdate) DeepCopy() *RollingUpdate {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RouteSpec) DeepCopyInto(out *RouteSpec) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteSpec.
+func (in *RouteSpec) DeepCopy() *RouteSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(RouteSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SSHCredential) DeepCopyInto(out *SSHCredential) {
 	*out = *in

pkg/apis/kops/validation/aws.go

@@ -48,6 +48,18 @@ func awsValidateCluster(c *kops.Cluster) field.ErrorList {
 		allErrs = append(allErrs, awsValidateIAMAuthenticator(field.NewPath("spec", "authentication", "aws"), c.Spec.Authentication.AWS)...)
 	}
 
+	for i, subnet := range c.Spec.Subnets {
+		f := field.NewPath("spec", "subnets").Index(i)
+		if subnet.AdditionalRoutes != nil {
+			if len(subnet.ProviderID) > 0 {
+				allErrs = append(allErrs, field.Invalid(f, subnet, "additional routes cannot be added if the subnet is shared"))
+			} else if subnet.Type != kops.SubnetTypePrivate {
+				allErrs = append(allErrs, field.Invalid(f, subnet, "additional routes can only be added on private subnets"))
+			}
+			allErrs = append(allErrs, awsValidateAdditionalRoutes(f.Child("additionalRoutes"), subnet.AdditionalRoutes, c.Spec.NetworkCIDR)...)
+		}
+	}
+
 	return allErrs
 }
 
@@ -347,3 +359,48 @@ func hasAWSEBSCSIDriver(c kops.ClusterSpec) bool {
 
 	return *c.CloudConfig.AWSEBSCSIDriver.Enabled
 }
+
+func awsValidateAdditionalRoutes(fieldPath *field.Path, routes []kops.RouteSpec, cidr string) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	_, clusterNet, errClusterNet := net.ParseCIDR(cidr)
+	if errClusterNet != nil {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "networkCIDR"), cidr, "Invalid cluster cidr"))
+	} else {
+		for i, r := range routes {
+			f := fieldPath.Index(i)
+
+			// Check if target is a known type
+			if !strings.HasPrefix(r.Target, "pcx-") &&
+				!strings.HasPrefix(r.Target, "i-") &&
+				!strings.HasPrefix(r.Target, "nat-") &&
+				!strings.HasPrefix(r.Target, "tgw-") &&
+				!strings.HasPrefix(r.Target, "igw-") &&
+				!strings.HasPrefix(r.Target, "eigw-") {
+				allErrs = append(allErrs, field.Invalid(f.Child("target"), r, "unknown target type for route"))
+			}
+
+			ipRoute, _, e := net.ParseCIDR(r.CIDR)
+			if e != nil {
+				allErrs = append(allErrs, field.Invalid(f.Child("cidr"), r, "invalid cidr"))
+			} else if clusterNet.Contains(ipRoute) && strings.HasPrefix(r.Target, "pcx-") {
+				allErrs = append(allErrs, field.Forbidden(f.Child("target"), "target is more specific than cluster CIDR block. This route can target only an interface or an instance."))
+			}
+		}
+	}
+
+	// Check for duplicated CIDR
+	{
+		cidrs := sets.NewString()
+		cidrs.Insert(cidr)
+		for i := range routes {
+			rCidr := routes[i].CIDR
+			if cidrs.Has(rCidr) {
+				allErrs = append(allErrs, field.Duplicate(fieldPath.Index(i).Child("cidr"), rCidr))
+			}
+			cidrs.Insert(rCidr)
+		}
+	}
+
+	return allErrs
+}

pkg/apis/kops/validation/aws_test.go

@@ -644,3 +644,173 @@ func TestAWSAuthentication(t *testing.T) {
 		testErrors(t, test, errs, test.expected)
 	}
 }
+
+func TestAWSAdditionalRoutes(t *testing.T) {
+	tests := []struct {
+		clusterCidr string
+		providerId  string
+		subnetType  kops.SubnetType
+		route       []kops.RouteSpec
+		expected    []string
+	}{
+		{ // valid pcx
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "pcx-abcdef",
+				},
+			},
+		},
+		{ // valid instance
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "i-abcdef",
+				},
+			},
+		},
+		{ // valid nat
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "nat-abcdef",
+				},
+			},
+		},
+		{ // valid transit gateway
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "tgw-abcdef",
+				},
+			},
+		},
+		{ // valid internet gateway
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "igw-abcdef",
+				},
+			},
+		},
+		{ // valid egress only internet gateway
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "eigw-abcdef",
+				},
+			},
+		},
+		{ // bad cluster cidr
+			clusterCidr: "not cidr",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "pcx-abcdef",
+				},
+			},
+			expected: []string{"Invalid value::spec.networkCIDR"},
+		},
+		{ // bad cidr
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "bad cidr",
+					Target: "pcx-abcdef",
+				},
+			},
+			expected: []string{"Invalid value::spec.subnets[0].additionalRoutes[0].cidr"},
+		},
+		{ // bad target
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "unknown-abcdef",
+				},
+			},
+			expected: []string{"Invalid value::spec.subnets[0].additionalRoutes[0].target"},
+		},
+		{ // target more specific
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "100.64.1.0/24",
+					Target: "pcx-abcdef",
+				},
+			},
+			expected: []string{"Forbidden::spec.subnets[0].additionalRoutes[0].target"},
+		},
+		{ // duplicates cidr
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "pcx-abcdef",
+				},
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "tgw-abcdef",
+				},
+			},
+			expected: []string{"Duplicate value::spec.subnets[0].additionalRoutes[1].cidr"},
+		},
+		{ // shared subnet
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePrivate,
+			providerId:  "123456",
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "pcx-abcdef",
+				},
+			},
+			expected: []string{"Invalid value::spec.subnets[0]"},
+		},
+		{ // not a private subnet
+			clusterCidr: "100.64.0.0/10",
+			subnetType:  kops.SubnetTypePublic,
+			route: []kops.RouteSpec{
+				{
+					CIDR:   "10.0.0.0/8",
+					Target: "pcx-abcdef",
+				},
+			},
+			expected: []string{"Invalid value::spec.subnets[0]"},
+		},
+	}
+
+	for _, test := range tests {
+		cluster := kops.Cluster{
+			Spec: kops.ClusterSpec{
+				NetworkCIDR: test.clusterCidr,
+				Subnets: []kops.ClusterSubnetSpec{
+					{
+						ProviderID:       test.providerId,
+						Type:             test.subnetType,
+						AdditionalRoutes: test.route,
+					},
+				},
+			},
+		}
+		errs := awsValidateCluster(&cluster)
+		testErrors(t, test, errs, test.expected)
+	}
+}

pkg/apis/kops/zz_generated.deepcopy.go

@@ -1054,7 +1054,9 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 	if in.Subnets != nil {
 		in, out := &in.Subnets, &out.Subnets
 		*out = make([]ClusterSubnetSpec, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	if in.AdditionalNetworkCIDRs != nil {
 		in, out := &in.AdditionalNetworkCIDRs, &out.AdditionalNetworkCIDRs
@@ -1393,6 +1395,11 @@ func (in *ClusterStatus) DeepCopy() *ClusterStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClusterSubnetSpec) DeepCopyInto(out *ClusterSubnetSpec) {
 	*out = *in
+	if in.AdditionalRoutes != nil {
+		in, out := &in.AdditionalRoutes, &out.AdditionalRoutes
+		*out = make([]RouteSpec, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -5105,6 +5112,22 @@ func (in *RomanaNetworkingSpec) DeepCopy() *RomanaNetworkingSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RouteSpec) DeepCopyInto(out *RouteSpec) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteSpec.
+func (in *RouteSpec) DeepCopy() *RouteSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(RouteSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SSHCredential) DeepCopyInto(out *SSHCredential) {
 	*out = *in

pkg/model/awsmodel/context.go

@@ -80,3 +80,27 @@ func (b *AWSModelContext) LinkToUtilitySubnetInZone(zoneName string) (*awstasks.
 
 	return b.LinkToSubnet(matches[0]), nil
 }
+func (b *AWSModelContext) LinkToPrivateSubnetsInZone(zoneName string) ([]*awstasks.Subnet, error) {
+	var matches []*kops.ClusterSubnetSpec
+	for i := range b.Cluster.Spec.Subnets {
+		s := &b.Cluster.Spec.Subnets[i]
+		if s.Zone != zoneName {
+			continue
+		}
+		if s.Type != kops.SubnetTypePrivate {
+			continue
+		}
+		matches = append(matches, s)
+	}
+	if len(matches) == 0 {
+		return nil, fmt.Errorf("could not find private subnet in zone: %q", zoneName)
+	}
+
+	var subnets []*awstasks.Subnet
+
+	for _, match := range matches {
+		subnets = append(subnets, b.LinkToSubnet(match))
+	}
+
+	return subnets, nil
+}

pkg/model/awsmodel/network.go

@@ -564,6 +564,22 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 					EgressOnlyInternetGateway: eigw,
 				})
 			}
+
+			subnets, err := b.LinkToPrivateSubnetsInZone(zone)
+			if err != nil {
+				return err
+			}
+
+			for _, subnetSpec := range b.Cluster.Spec.Subnets {
+				for _, subnet := range subnets {
+					if strings.HasPrefix(*subnet.Name, subnetSpec.Name) {
+						err := addAdditionalRoutes(subnetSpec.AdditionalRoutes, subnetSpec.Name, rt, b.Lifecycle, c)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
 		}
 
 		if info.HaveIPv6PublicSubnet {
@@ -615,3 +631,74 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 
 	return nil
 }
+
+func addAdditionalRoutes(routes []kops.RouteSpec, sbName string, rt *awstasks.RouteTable, lf fi.Lifecycle, c *fi.ModelBuilderContext) error {
+	for _, r := range routes {
+		t := &awstasks.Route{
+			Name:       fi.String(sbName + "." + r.CIDR),
+			Lifecycle:  lf,
+			CIDR:       fi.String(r.CIDR),
+			RouteTable: rt,
+		}
+		if strings.HasPrefix(r.Target, "pcx-") {
+			t.VPCPeeringConnectionID = fi.String(r.Target)
+			c.AddTask(t)
+		} else if strings.HasPrefix(r.Target, "i-") {
+			inst := &awstasks.Instance{
+				Name:      fi.String(r.Target),
+				Lifecycle: lf,
+				ID:        fi.String(r.Target),
+				Shared:    fi.Bool(true),
+			}
+			err := c.EnsureTask(inst)
+			if err != nil {
+				return err
+			}
+			t.Instance = inst
+			c.AddTask(t)
+		} else if strings.HasPrefix(r.Target, "nat-") {
+			nat := &awstasks.NatGateway{
+				Name:      fi.String(r.Target),
+				Lifecycle: lf,
+				ID:        fi.String(r.Target),
+				Shared:    fi.Bool(true),
+			}
+			err := c.EnsureTask(nat)
+			if err != nil {
+				return err
+			}
+			t.NatGateway = nat
+			c.AddTask(t)
+		} else if strings.HasPrefix(r.Target, "tgw-") {
+			t.TransitGatewayID = fi.String(r.Target)
+			c.AddTask(t)
+		} else if strings.HasPrefix(r.Target, "igw-") {
+			internetGW := &awstasks.InternetGateway{
+				Name:      fi.String(r.Target),
+				Lifecycle: lf,
+				ID:        fi.String(r.Target),
+				Shared:    fi.Bool(true),
+			}
+			err := c.EnsureTask(internetGW)
+			if err != nil {
+				return err
+			}
+			t.InternetGateway = internetGW
+			c.AddTask(t)
+		} else if strings.HasPrefix(r.Target, "eigw-") {
+			eigw := &awstasks.EgressOnlyInternetGateway{
+				Name:      fi.String(r.Target),
+				Lifecycle: lf,
+				ID:        fi.String(r.Target),
+				Shared:    fi.Bool(true),
+			}
+			err := c.EnsureTask(eigw)
+			if err != nil {
+				return err
+			}
+			t.EgressOnlyInternetGateway = eigw
+			c.AddTask(t)
+		}
+	}
+	return nil
+}

tests/integration/update_cluster/complex/cloudformation.json

@@ -673,6 +673,16 @@
         "TransitGatewayId": "tgw-123456"
       }
     },
+    "AWSEC2Routeuseast1aprivate19216811032": {
+      "Type": "AWS::EC2::Route",
+      "Properties": {
+        "RouteTableId": {
+          "Ref": "AWSEC2RouteTableprivateustest1acomplexexamplecom"
+        },
+        "DestinationCidrBlock": "192.168.1.10/32",
+        "TransitGatewayId": "tgw-0123456"
+      }
+    },
     "AWSEC2SecurityGroupEgressfrommasterscomplexexamplecomegressall0to00": {
       "Type": "AWS::EC2::SecurityGroupEgress",
       "Properties": {

tests/integration/update_cluster/complex/data/aws_s3_bucket_object_cluster-completed.spec_content

@@ -206,7 +206,10 @@ spec:
     name: us-test-1a
     type: Public
     zone: us-test-1a
-  - cidr: 172.20.64.0/19
+  - additionalRoutes:
+    - cidr: 192.168.1.10/32
+      target: tgw-0123456
+    cidr: 172.20.64.0/19
     egress: tgw-123456
     name: us-east-1a-private
     type: Private

tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml

@@ -88,6 +88,9 @@ spec:
     type: Private
     zone: us-test-1a
     egress: tgw-123456
+    additionalRoutes:
+    - cidr: "192.168.1.10/32"
+      target: "tgw-0123456"
   - cidr: 172.20.96.0/19
     name: us-east-1a-utility
     type: Utility

tests/integration/update_cluster/complex/in-v1alpha2.yaml

@@ -89,6 +89,9 @@ spec:
     type: Private
     zone: us-test-1a
     egress: tgw-123456
+    additionalRoutes:
+    - cidr: "192.168.1.10/32"
+      target: "tgw-0123456"
   - cidr: 172.20.96.0/19
     name: us-east-1a-utility
     type: Utility

tests/integration/update_cluster/complex/kubernetes.tf

@@ -632,6 +632,12 @@ resource "aws_route" "route-private-us-test-1a-0-0-0-0--0" {
   transit_gateway_id     = "tgw-123456"
 }
 
+resource "aws_route" "route-us-east-1a-private-192-168-1-10--32" {
+  destination_cidr_block = "192.168.1.10/32"
+  route_table_id         = aws_route_table.private-us-test-1a-complex-example-com.id
+  transit_gateway_id     = "tgw-0123456"
+}
+
 resource "aws_route53_record" "api-complex-example-com" {
   alias {
     evaluate_target_health = false

upup/pkg/fi/cloudup/awstasks/route.go

@@ -45,6 +45,7 @@ type Route struct {
 	InternetGateway           *InternetGateway
 	NatGateway                *NatGateway
 	TransitGatewayID          *string
+	VPCPeeringConnectionID    *string
 }
 
 func (e *Route) Find(c *fi.Context) (*Route, error) {
@@ -100,6 +101,9 @@ func (e *Route) Find(c *fi.Context) (*Route, error) {
 			if r.TransitGatewayId != nil {
 				actual.TransitGatewayID = r.TransitGatewayId
 			}
+			if r.VpcPeeringConnectionId != nil {
+				actual.VPCPeeringConnectionID = r.VpcPeeringConnectionId
+			}
 
 			if aws.StringValue(r.State) == "blackhole" {
 				klog.V(2).Infof("found route is a blackhole route")
@@ -155,11 +159,14 @@ func (s *Route) CheckChanges(a, e, changes *Route) error {
 		if e.TransitGatewayID != nil {
 			targetCount++
 		}
+		if e.VPCPeeringConnectionID != nil {
+			targetCount++
+		}
 		if targetCount == 0 {
-			return fmt.Errorf("EgressOnlyInternetGateway, InternetGateway, Instance, NatGateway, or TransitGateway is required")
+			return fmt.Errorf("EgressOnlyInternetGateway, InternetGateway, Instance, NatGateway, TransitGateway, or VpcPeeringConnection is required")
 		}
 		if targetCount != 1 {
-			return fmt.Errorf("cannot set more than one EgressOnlyInternetGateway, InternetGateway, Instance, NatGateway, or TransitGateway")
+			return fmt.Errorf("cannot set more than one EgressOnlyInternetGateway, InternetGateway, Instance, NatGateway, TransitGateway, or VpcPeeringConnection")
 		}
 	}
 
@@ -189,7 +196,7 @@ func (_ *Route) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Route) error {
 			klog.Fatal("both CIDR and IPv6CIDR were unexpectedly nil")
 		}
 
-		if e.EgressOnlyInternetGateway == nil && e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil {
+		if e.EgressOnlyInternetGateway == nil && e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil && e.VPCPeeringConnectionID == nil {
 			return fmt.Errorf("missing target for route")
 		} else if e.EgressOnlyInternetGateway != nil {
 			request.EgressOnlyInternetGatewayId = checkNotNil(e.EgressOnlyInternetGateway.ID)
@@ -199,6 +206,8 @@ func (_ *Route) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Route) error {
 			request.NatGatewayId = checkNotNil(e.NatGateway.ID)
 		} else if e.TransitGatewayID != nil {
 			request.TransitGatewayId = e.TransitGatewayID
+		} else if e.VPCPeeringConnectionID != nil {
+			request.VpcPeeringConnectionId = e.VPCPeeringConnectionID
 		}
 
 		if e.Instance != nil {
@@ -233,7 +242,7 @@ func (_ *Route) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Route) error {
 			klog.Fatal("both CIDR and IPv6CIDR were unexpectedly nil")
 		}
 
-		if e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil {
+		if e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil && e.VPCPeeringConnectionID == nil {
 			return fmt.Errorf("missing target for route")
 		} else if e.InternetGateway != nil {
 			request.GatewayId = checkNotNil(e.InternetGateway.ID)
@@ -241,6 +250,8 @@ func (_ *Route) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Route) error {
 			request.NatGatewayId = checkNotNil(e.NatGateway.ID)
 		} else if e.TransitGatewayID != nil {
 			request.TransitGatewayId = e.TransitGatewayID
+		} else if e.VPCPeeringConnectionID != nil {
+			request.VpcPeeringConnectionId = e.VPCPeeringConnectionID
 		}
 
 		if e.Instance != nil {
@@ -279,6 +290,7 @@ type terraformRoute struct {
 	NATGatewayID                *terraformWriter.Literal `cty:"nat_gateway_id"`
 	TransitGatewayID            *string                  `cty:"transit_gateway_id"`
 	InstanceID                  *terraformWriter.Literal `cty:"instance_id"`
+	VPCPeeringConnectionID      *string                  `cty:"vpc_peering_connection_id"`
 }
 
 func (_ *Route) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Route) error {
@@ -288,7 +300,7 @@ func (_ *Route) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Rou
 		IPv6CIDR:     e.IPv6CIDR,
 	}
 
-	if e.EgressOnlyInternetGateway == nil && e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil {
+	if e.EgressOnlyInternetGateway == nil && e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil && e.VPCPeeringConnectionID == nil {
 		return fmt.Errorf("missing target for route")
 	} else if e.EgressOnlyInternetGateway != nil {
 		tf.EgressOnlyInternetGatewayID = e.EgressOnlyInternetGateway.TerraformLink()
@@ -298,6 +310,8 @@ func (_ *Route) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Rou
 		tf.NATGatewayID = e.NatGateway.TerraformLink()
 	} else if e.TransitGatewayID != nil {
 		tf.TransitGatewayID = e.TransitGatewayID
+	} else if e.VPCPeeringConnectionID != nil {
+		tf.VPCPeeringConnectionID = e.VPCPeeringConnectionID
 	}
 
 	if e.Instance != nil {
@@ -311,13 +325,14 @@ func (_ *Route) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Rou
 }
 
 type cloudformationRoute struct {
-	RouteTableID      *cloudformation.Literal `json:"RouteTableId"`
-	CIDR              *string                 `json:"DestinationCidrBlock,omitempty"`
-	IPv6CIDR          *string                 `json:"DestinationIpv6CidrBlock,omitempty"`
-	InternetGatewayID *cloudformation.Literal `json:"GatewayId,omitempty"`
-	NATGatewayID      *cloudformation.Literal `json:"NatGatewayId,omitempty"`
-	TransitGatewayID  *string                 `json:"TransitGatewayId,omitempty"`
-	InstanceID        *cloudformation.Literal `json:"InstanceId,omitempty"`
+	RouteTableID           *cloudformation.Literal `json:"RouteTableId"`
+	CIDR                   *string                 `json:"DestinationCidrBlock,omitempty"`
+	IPv6CIDR               *string                 `json:"DestinationIpv6CidrBlock,omitempty"`
+	InternetGatewayID      *cloudformation.Literal `json:"GatewayId,omitempty"`
+	NATGatewayID           *cloudformation.Literal `json:"NatGatewayId,omitempty"`
+	TransitGatewayID       *string                 `json:"TransitGatewayId,omitempty"`
+	InstanceID             *cloudformation.Literal `json:"InstanceId,omitempty"`
+	VPCPeeringConnectionID *string                 `json:"VpcPeeringConnectionId,omitempty"`
 }
 
 func (_ *Route) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *Route) error {
@@ -327,7 +342,7 @@ func (_ *Route) RenderCloudformation(t *cloudformation.CloudformationTarget, a,
 		IPv6CIDR:     e.IPv6CIDR,
 	}
 
-	if e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil {
+	if e.InternetGateway == nil && e.NatGateway == nil && e.TransitGatewayID == nil && e.VPCPeeringConnectionID == nil {
 		return fmt.Errorf("missing target for route")
 	} else if e.InternetGateway != nil {
 		tf.InternetGatewayID = e.InternetGateway.CloudformationLink()
@@ -335,6 +350,8 @@ func (_ *Route) RenderCloudformation(t *cloudformation.CloudformationTarget, a,
 		tf.NATGatewayID = e.NatGateway.CloudformationLink()
 	} else if e.TransitGatewayID != nil {
 		tf.TransitGatewayID = e.TransitGatewayID
+	} else if e.VPCPeeringConnectionID != nil {
+		tf.VPCPeeringConnectionID = e.VPCPeeringConnectionID
 	}
 
 	if e.Instance != nil {