From 7cb4fbe91e4229089e2aed24d5b781af92a87224 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Wed, 27 Oct 2021 20:02:30 -0700 Subject: [PATCH 1/2] Never masquerade IPv6 with Cilium --- nodeup/pkg/model/prefix.go | 2 +- pkg/apis/kops/cluster.go | 7 ++++--- pkg/apis/kops/v1alpha2/cluster.go | 3 --- pkg/apis/kops/v1alpha3/cluster.go | 3 --- pkg/apis/kops/validation/validation.go | 8 -------- pkg/model/components/cilium.go | 2 +- pkg/model/components/kubecontrollermanager.go | 6 +++--- pkg/model/components/kubeproxy.go | 2 +- pkg/model/iam/iam_builder.go | 2 +- .../ipv6-cloudipam/in-v1alpha2.yaml | 3 +-- .../k8s-1.16-v1.10.yaml.template | 6 +++--- upup/pkg/fi/cloudup/populate_cluster_spec.go | 10 ++-------- .../fi/cloudup/populate_cluster_spec_test.go | 20 ++----------------- upup/pkg/fi/cloudup/template_functions.go | 2 +- 14 files changed, 20 insertions(+), 56 deletions(-) diff --git a/nodeup/pkg/model/prefix.go b/nodeup/pkg/model/prefix.go index 142d2664c2..c042e56d33 100644 --- a/nodeup/pkg/model/prefix.go +++ b/nodeup/pkg/model/prefix.go @@ -28,7 +28,7 @@ type PrefixBuilder struct { var _ fi.ModelBuilder = &PrefixBuilder{} func (b *PrefixBuilder) Build(c *fi.ModelBuilderContext) error { - if !b.Cluster.Spec.PodCIDRFromCloud { + if !b.Cluster.Spec.IsKopsControllerIPAM() { return nil } c.AddTask(&nodetasks.Prefix{ diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index f8670f8dc9..fd8e2177ad 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -112,9 +112,6 @@ type ClusterSpec struct { ServiceClusterIPRange string `json:"serviceClusterIPRange,omitempty"` // PodCIDR is the CIDR from which we allocate IPs for pods PodCIDR string `json:"podCIDR,omitempty"` - // PodCIDRFromCloud determines if the Node's podCIDR should be set by the cloud provider. - // This requires ipv6 enabled and that instances can be given full ipv6 prefixes. - PodCIDRFromCloud bool `json:"podCIDRFromCloud,omitempty"` // NonMasqueradeCIDR is the CIDR for the internal k8s network (on which pods & services live) // It cannot overlap ServiceClusterIPRange NonMasqueradeCIDR string `json:"nonMasqueradeCIDR,omitempty"` @@ -824,6 +821,10 @@ func (c *ClusterSpec) IsIPv6Only() bool { return utils.IsIPv6CIDR(c.NonMasqueradeCIDR) } +func (c *ClusterSpec) IsKopsControllerIPAM() bool { + return c.IsIPv6Only() +} + // EnvVar represents an environment variable present in a Container. type EnvVar struct { // Name of the environment variable. Must be a C_IDENTIFIER. diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 394e854978..2e118b5f7c 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -107,9 +107,6 @@ type ClusterSpec struct { ServiceClusterIPRange string `json:"serviceClusterIPRange,omitempty"` // PodCIDR is the CIDR from which we allocate IPs for pods PodCIDR string `json:"podCIDR,omitempty"` - // PodCIDRFromCloud determines if the Node's podCIDR should be set by the cloud provider. - // This requires ipv6 enabled and that instances can be given full ipv6 prefixes. - PodCIDRFromCloud bool `json:"podCIDRFromCloud,omitempty"` //MasterIPRange string `json:",omitempty"` // NonMasqueradeCIDR is the CIDR for the internal k8s network (on which pods & services live) // It cannot overlap ServiceClusterIPRange diff --git a/pkg/apis/kops/v1alpha3/cluster.go b/pkg/apis/kops/v1alpha3/cluster.go index 3eaa74ef15..34cf538d51 100644 --- a/pkg/apis/kops/v1alpha3/cluster.go +++ b/pkg/apis/kops/v1alpha3/cluster.go @@ -108,9 +108,6 @@ type ClusterSpec struct { ServiceClusterIPRange string `json:"serviceClusterIPRange,omitempty"` // PodCIDR is the CIDR from which we allocate IPs for pods PodCIDR string `json:"podCIDR,omitempty"` - // PodCIDRFromCloud determines if the Node's podCIDR should be set by the cloud provider. - // This requires ipv6 enabled and that instances can be given full ipv6 prefixes. - PodCIDRFromCloud bool `json:"podCIDRFromCloud,omitempty"` //MasterIPRange string `json:",omitempty"` // NonMasqueradeCIDR is the CIDR for the internal k8s network (on which pods & services live) // It cannot overlap ServiceClusterIPRange diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index f0b370d19b..8257d34130 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -36,7 +36,6 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/dns" - "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model/components" "k8s.io/kops/pkg/model/iam" "k8s.io/kops/upup/pkg/fi" @@ -263,13 +262,6 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie } } - if spec.PodCIDRFromCloud { - if !featureflag.AWSIPv6.Enabled() { - allErrs = append(allErrs, field.Forbidden(fieldPath.Child("podCIDRFromCloud", "serviceAccountExternalPermissions"), "podCIDRFromCloud requires the AWSIPv6 feature flag to be enabled")) - - } - } - return allErrs } diff --git a/pkg/model/components/cilium.go b/pkg/model/components/cilium.go index 6e37104282..ff02f639fc 100644 --- a/pkg/model/components/cilium.go +++ b/pkg/model/components/cilium.go @@ -116,7 +116,7 @@ func (b *CiliumOptionsBuilder) BuildOptions(o interface{}) error { } if c.Tunnel == "" { - if c.Ipam == "eni" || clusterSpec.PodCIDRFromCloud { + if c.Ipam == "eni" || clusterSpec.IsIPv6Only() { c.Tunnel = "disabled" } else { c.Tunnel = "vxlan" diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go index b21c76ec1b..e7d2bd9a9d 100644 --- a/pkg/model/components/kubecontrollermanager.go +++ b/pkg/model/components/kubecontrollermanager.go @@ -116,9 +116,9 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error // Doesn't seem to be any real downside to always doing a leader election kcm.LeaderElection = &kops.LeaderElectionConfiguration{LeaderElect: fi.Bool(true)} - kcm.AllocateNodeCIDRs = fi.Bool(!clusterSpec.PodCIDRFromCloud) + kcm.AllocateNodeCIDRs = fi.Bool(!clusterSpec.IsKopsControllerIPAM()) - if kcm.ClusterCIDR == "" && !clusterSpec.PodCIDRFromCloud { + if kcm.ClusterCIDR == "" && !clusterSpec.IsKopsControllerIPAM() { kcm.ClusterCIDR = clusterSpec.PodCIDR } @@ -163,7 +163,7 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error if fi.BoolValue(clusterSpec.KubeAPIServer.EnableBootstrapAuthToken) { changes = append(changes, "tokencleaner") } - if clusterSpec.PodCIDRFromCloud { + if clusterSpec.IsKopsControllerIPAM() { changes = append(changes, "-nodeipam") } if len(changes) != 0 { diff --git a/pkg/model/components/kubeproxy.go b/pkg/model/components/kubeproxy.go index 1c0c5cf558..87d65e59db 100644 --- a/pkg/model/components/kubeproxy.go +++ b/pkg/model/components/kubeproxy.go @@ -99,7 +99,7 @@ func (b *KubeProxyOptionsBuilder) BuildOptions(o interface{}) error { func (*KubeProxyOptionsBuilder) needsClusterCIDR(clusterSpec *kops.ClusterSpec) bool { // If we use podCIDR from cloud, we should not set cluster cidr. - if clusterSpec.PodCIDRFromCloud { + if clusterSpec.IsKopsControllerIPAM() { return false } diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 97647b9cc5..d3414e42fb 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -768,7 +768,7 @@ func (b *PolicyBuilder) addNodeupPermissions(p *Policy, enableHookSupport bool) "ec2:DescribeInstanceTypes", ) - if b.Cluster.Spec.PodCIDRFromCloud { + if b.Cluster.Spec.IsKopsControllerIPAM() { p.unconditionalAction.Insert( "ec2:DescribeNetworkInterfaces", "ec2:AssignIpv6Addresses", diff --git a/tests/integration/update_cluster/ipv6-cloudipam/in-v1alpha2.yaml b/tests/integration/update_cluster/ipv6-cloudipam/in-v1alpha2.yaml index a98b4e3f29..b4ae269ff0 100644 --- a/tests/integration/update_cluster/ipv6-cloudipam/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/ipv6-cloudipam/in-v1alpha2.yaml @@ -35,8 +35,7 @@ spec: networkCIDR: 172.20.0.0/16 networking: cni: {} - nonMasqueradeCIDR: 100.64.0.0/10 - podCIDRFromCloud: true + nonMasqueradeCIDR: ::/0 topology: masters: public nodes: public diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template index d082107c75..91ef583eba 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template @@ -151,7 +151,7 @@ data: # enable-bpf-masquerade enables masquerading packets from endpoints leaving # the host with BPF instead of iptables. (default false) - enable-bpf-masquerade: "{{ .EnableBPFMasquerade }}" + enable-bpf-masquerade: "{{ and (WithDefaultBool .EnableBPFMasquerade false) (not IsIPv6Only) }}" # Pre-allocation of map entries allows per-packet latency to be reduced, at # the expense of up-front memory allocation for the entries in the maps. The @@ -221,8 +221,8 @@ data: # - none # - auto (automatically detect the container runtime) # - masquerade: "{{- if WithDefaultBool .DisableMasquerade false -}}false{{- else -}}true{{- end -}}" - enable-ipv6-masquerade: "{{- if WithDefaultBool .DisableMasquerade false -}}false{{- else -}}true{{- end -}}" + masquerade: "{{- not (or IsIPv6Only (WithDefaultBool .DisableMasquerade false) ) -}}" + enable-ipv6-masquerade: "false" install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}" auto-direct-node-routes: "{{ .AutoDirectNodeRoutes }}" {{ if .EnableHostReachableServices }} diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index 94b69d0321..2faad7d68b 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -326,15 +326,9 @@ func (c *populateClusterSpec) assignSubnets(cluster *kopsapi.Cluster) error { cluster.Spec.KubeControllerManager = &kopsapi.KubeControllerManagerConfig{} } - if cluster.Spec.PodCIDR == "" && nmOnes > 0 { + if cluster.Spec.PodCIDR == "" && nmBits == 32 { // Allocate as big a range as possible: the NonMasqueradeCIDR mask + 1, with a '1' in the extra bit ip := nonMasqueradeCIDR.IP.Mask(nonMasqueradeCIDR.Mask) - if nmBits > 32 && nmOnes < 95 { - // The maximum size of an IPv6 ClusterCIDR is /64, but a /112 node CIDR gives far more addresses - // than Kubernetes can handle on a node and is more visually pleasing. - // Technically, the maximum size of an IPv4 ClusterCIDR is /8, but nobody has a /7 to allocate. - nmOnes = 95 - } ip[nmOnes/8] |= 128 >> (nmOnes % 8) cidr := net.IPNet{IP: ip, Mask: net.CIDRMask(nmOnes+1, nmBits)} cluster.Spec.PodCIDR = cidr.String() @@ -342,7 +336,7 @@ func (c *populateClusterSpec) assignSubnets(cluster *kopsapi.Cluster) error { } if cluster.Spec.ServiceClusterIPRange == "" { - if nmBits > 32 && nmOnes == 0 { + if nmBits > 32 { cluster.Spec.ServiceClusterIPRange = "fd00:5e4f:ce::/108" } else { // Allocate from the '0' subnet; but only carve off 1/4 of that (i.e. add 1 + 2 bits to the netmask) diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec_test.go b/upup/pkg/fi/cloudup/populate_cluster_spec_test.go index 31bed24999..904ee6455f 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec_test.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec_test.go @@ -77,24 +77,8 @@ func TestPopulateCluster_Subnets(t *testing.T) { ExpectedServiceClusterIPRange: "10.0.0.0/12", }, { - NonMasqueradeCIDR: "fd00:10:96::/96", - ExpectedClusterCIDR: "fd00:10:96::8000:0/97", - ExpectedServiceClusterIPRange: "fd00:10:96::/108", - }, - { - NonMasqueradeCIDR: "fd00:10:96::/95", - ExpectedClusterCIDR: "fd00:10:96::1:0:0/96", - ExpectedServiceClusterIPRange: "fd00:10:96::/108", - }, - { - NonMasqueradeCIDR: "fd00:10:96::/94", - ExpectedClusterCIDR: "fd00:10:96::1:0:0/96", - ExpectedServiceClusterIPRange: "fd00:10:96::/108", - }, - { - NonMasqueradeCIDR: "fd00:10:96::/106", - ExpectedClusterCIDR: "fd00:10:96::20:0/107", - ExpectedServiceClusterIPRange: "fd00:10:96::/109", + NonMasqueradeCIDR: "::/0", + ExpectedServiceClusterIPRange: "fd00:5e4f:ce::/108", }, } for _, tc := range tests { diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 4eca689db4..e911bf2e39 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -569,7 +569,7 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { } } - if tf.Cluster.Spec.PodCIDRFromCloud { + if tf.Cluster.Spec.IsKopsControllerIPAM() { config.EnableCloudIPAM = true } From 7b5fdc7c43276b9c11328354a2821ec6e8166969 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Wed, 27 Oct 2021 22:57:03 -0700 Subject: [PATCH 2/2] Update automatically generated files --- k8s/crds/kops.k8s.io_clusters.yaml | 5 ---- .../kops/v1alpha2/zz_generated.conversion.go | 2 -- .../kops/v1alpha3/zz_generated.conversion.go | 2 -- ...masters.minimal-ipv6.example.com_user_data | 17 +++++++------ ...e_nodes.minimal-ipv6.example.com_user_data | 9 ++++--- ...cket_object_cluster-completed.spec_content | 25 +++++++++++-------- ...-ipv6.example.com-addons-bootstrap_content | 2 +- ...ons-coredns.addons.k8s.io-k8s-1.12_content | 4 +-- ...ect_nodeupconfig-master-us-test-1a_content | 8 +++--- ...3_bucket_object_nodeupconfig-nodes_content | 4 +-- ...mpool.example.com-addons-bootstrap_content | 2 +- ...dons-networking.cilium.io-k8s-1.16_content | 2 +- ...ilium.example.com-addons-bootstrap_content | 2 +- ...dons-networking.cilium.io-k8s-1.16_content | 2 +- .../cilium/manifest.yaml | 2 +- .../insecure-1.18/manifest.yaml | 2 +- .../insecure-1.19/manifest.yaml | 2 +- .../metrics-server/secure-1.18/manifest.yaml | 2 +- .../metrics-server/secure-1.19/manifest.yaml | 2 +- 19 files changed, 48 insertions(+), 48 deletions(-) diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 3e01814733..d5cf5c7ccd 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -5197,11 +5197,6 @@ spec: podCIDR: description: PodCIDR is the CIDR from which we allocate IPs for pods type: string - podCIDRFromCloud: - description: PodCIDRFromCloud determines if the Node's podCIDR should - be set by the cloud provider. This requires ipv6 enabled and that - instances can be given full ipv6 prefixes. - type: boolean project: description: Project is the cloud project we should use, required on GCE diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index a4426a9e75..8f2bf7e790 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2377,7 +2377,6 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.ClusterDNSDomain = in.ClusterDNSDomain out.ServiceClusterIPRange = in.ServiceClusterIPRange out.PodCIDR = in.PodCIDR - out.PodCIDRFromCloud = in.PodCIDRFromCloud out.NonMasqueradeCIDR = in.NonMasqueradeCIDR out.SSHAccess = in.SSHAccess out.NodePortAccess = in.NodePortAccess @@ -2790,7 +2789,6 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, out.ClusterDNSDomain = in.ClusterDNSDomain out.ServiceClusterIPRange = in.ServiceClusterIPRange out.PodCIDR = in.PodCIDR - out.PodCIDRFromCloud = in.PodCIDRFromCloud out.NonMasqueradeCIDR = in.NonMasqueradeCIDR out.SSHAccess = in.SSHAccess out.NodePortAccess = in.NodePortAccess diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index c284104559..657aa5bb65 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -2306,7 +2306,6 @@ func autoConvert_v1alpha3_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.ClusterDNSDomain = in.ClusterDNSDomain out.ServiceClusterIPRange = in.ServiceClusterIPRange out.PodCIDR = in.PodCIDR - out.PodCIDRFromCloud = in.PodCIDRFromCloud out.NonMasqueradeCIDR = in.NonMasqueradeCIDR out.SSHAccess = in.SSHAccess out.NodePortAccess = in.NodePortAccess @@ -2711,7 +2710,6 @@ func autoConvert_kops_ClusterSpec_To_v1alpha3_ClusterSpec(in *kops.ClusterSpec, out.ClusterDNSDomain = in.ClusterDNSDomain out.ServiceClusterIPRange = in.ServiceClusterIPRange out.PodCIDR = in.PodCIDR - out.PodCIDRFromCloud = in.PodCIDRFromCloud out.NonMasqueradeCIDR = in.NonMasqueradeCIDR out.SSHAccess = in.SSHAccess out.NodePortAccess = in.NodePortAccess diff --git a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data index 0e9e2d9c29..e745b0629e 100644 --- a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data +++ b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data @@ -127,6 +127,9 @@ cloudConfig: awsEBSCSIDriver: enabled: false manageStorageClasses: true + nodeIPFamilies: + - ipv6 + - ipv4 containerRuntime: containerd containerd: logLevel: info @@ -146,7 +149,7 @@ kubeAPIServer: - kubernetes.svc.default apiServerCount: 1 authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 + bindAddress: '::' cloudProvider: aws enableAdmissionPlugins: - NamespaceLifecycle @@ -179,7 +182,7 @@ kubeAPIServer: securePort: 443 serviceAccountIssuer: https://api.internal.minimal-ipv6.example.com serviceAccountJWKSURI: https://api.internal.minimal-ipv6.example.com/openid/v1/jwks - serviceClusterIPRange: 100.64.0.0/13 + serviceClusterIPRange: fd00:5e4f:ce::/108 storageBackend: etcd3 kubeControllerManager: allocateNodeCIDRs: false @@ -210,7 +213,7 @@ kubelet: cgroupDriver: systemd cgroupRoot: / cloudProvider: aws - clusterDNS: 100.64.0.10 + clusterDNS: fd00:5e4f:ce::a clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% @@ -218,14 +221,14 @@ kubelet: kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 networkPluginName: cni - nonMasqueradeCIDR: 100.64.0.0/10 + nonMasqueradeCIDR: ::/0 podManifestPath: /etc/kubernetes/manifests masterKubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / cloudProvider: aws - clusterDNS: 100.64.0.10 + clusterDNS: fd00:5e4f:ce::a clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% @@ -233,7 +236,7 @@ masterKubelet: kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 networkPluginName: cni - nonMasqueradeCIDR: 100.64.0.0/10 + nonMasqueradeCIDR: ::/0 podManifestPath: /etc/kubernetes/manifests registerSchedulable: false @@ -244,7 +247,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal-ipv6.example.com InstanceGroupName: master-us-test-1a InstanceGroupRole: Master -NodeupConfigHash: yrj4teDAp1g5GdFmTTsqdvkO9tGX3EVpiqHmwEPCLw4= +NodeupConfigHash: E/2UN3GhVLTRD3ByNw5y9wNYgvXfvosAyFhP1X5utsw= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data index 3398593ae3..cf2bfaa45d 100644 --- a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data +++ b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data @@ -127,6 +127,9 @@ cloudConfig: awsEBSCSIDriver: enabled: false manageStorageClasses: true + nodeIPFamilies: + - ipv6 + - ipv4 containerRuntime: containerd containerd: logLevel: info @@ -143,7 +146,7 @@ kubelet: cgroupDriver: systemd cgroupRoot: / cloudProvider: aws - clusterDNS: 100.64.0.10 + clusterDNS: fd00:5e4f:ce::a clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% @@ -151,7 +154,7 @@ kubelet: kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 networkPluginName: cni - nonMasqueradeCIDR: 100.64.0.0/10 + nonMasqueradeCIDR: ::/0 podManifestPath: /etc/kubernetes/manifests __EOF_CLUSTER_SPEC @@ -161,7 +164,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal-ipv6.example.com InstanceGroupName: nodes InstanceGroupRole: Node -NodeupConfigHash: oLOgAfNuXTV6ZrJSk0ddFu5+Jr/oeJ7LLMCtUQygY1w= +NodeupConfigHash: Mqfc35n7HWWI03aEiC/9tG99xKZd6sr0kJSCwJvzhKA= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_cluster-completed.spec_content b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_cluster-completed.spec_content index 430e6f43a9..e8a0745021 100644 --- a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_cluster-completed.spec_content @@ -15,6 +15,9 @@ spec: awsEBSCSIDriver: enabled: false manageStorageClasses: true + nodeIPFamilies: + - ipv6 + - ipv4 cloudProvider: aws clusterDNSDomain: cluster.local configBase: memfs://clusters.example.com/minimal-ipv6.example.com @@ -53,7 +56,7 @@ spec: - kubernetes.svc.default apiServerCount: 1 authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 + bindAddress: '::' cloudProvider: aws enableAdmissionPlugins: - NamespaceLifecycle @@ -86,7 +89,7 @@ spec: securePort: 443 serviceAccountIssuer: https://api.internal.minimal-ipv6.example.com serviceAccountJWKSURI: https://api.internal.minimal-ipv6.example.com/openid/v1/jwks - serviceClusterIPRange: 100.64.0.0/13 + serviceClusterIPRange: fd00:5e4f:ce::/108 storageBackend: etcd3 kubeControllerManager: allocateNodeCIDRs: false @@ -116,7 +119,9 @@ spec: memoryRequest: 5Mi provider: CoreDNS replicas: 2 - serverIP: 100.64.0.10 + serverIP: fd00:5e4f:ce::a + upstreamNameservers: + - fd00:ec2::253 kubeProxy: cpuRequest: 100m hostnameOverride: '@aws' @@ -132,7 +137,7 @@ spec: cgroupDriver: systemd cgroupRoot: / cloudProvider: aws - clusterDNS: 100.64.0.10 + clusterDNS: fd00:5e4f:ce::a clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% @@ -140,7 +145,7 @@ spec: kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 networkPluginName: cni - nonMasqueradeCIDR: 100.64.0.0/10 + nonMasqueradeCIDR: ::/0 podManifestPath: /etc/kubernetes/manifests kubernetesApiAccess: - 0.0.0.0/0 @@ -152,7 +157,7 @@ spec: cgroupDriver: systemd cgroupRoot: / cloudProvider: aws - clusterDNS: 100.64.0.10 + clusterDNS: fd00:5e4f:ce::a clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% @@ -160,18 +165,16 @@ spec: kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 networkPluginName: cni - nonMasqueradeCIDR: 100.64.0.0/10 + nonMasqueradeCIDR: ::/0 podManifestPath: /etc/kubernetes/manifests registerSchedulable: false masterPublicName: api.minimal-ipv6.example.com networkCIDR: 172.20.0.0/16 networking: cni: {} - nonMasqueradeCIDR: 100.64.0.0/10 - podCIDR: 100.96.0.0/11 - podCIDRFromCloud: true + nonMasqueradeCIDR: ::/0 secretStore: memfs://clusters.example.com/minimal-ipv6.example.com/secrets - serviceClusterIPRange: 100.64.0.0/13 + serviceClusterIPRange: fd00:5e4f:ce::/108 sshAccess: - 0.0.0.0/0 - ::/0 diff --git a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content index c9000ad629..9a8cedd338 100644 --- a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content @@ -20,7 +20,7 @@ spec: version: 9.99.0 - id: k8s-1.12 manifest: coredns.addons.k8s.io/k8s-1.12.yaml - manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef + manifestHash: e31327420b42b8d1b813625c65601166c52b054ae9ac95a57048d72e70b7033c name: coredns.addons.k8s.io selector: k8s-addon: coredns.addons.k8s.io diff --git a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content index 174ac503e8..e3271a983f 100644 --- a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content +++ b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content @@ -81,7 +81,7 @@ data: ttl 30 } prometheus :9153 - forward . /etc/resolv.conf { + forward . fd00:ec2::253 { max_concurrent 1000 } cache 30 @@ -226,7 +226,7 @@ metadata: namespace: kube-system resourceVersion: "0" spec: - clusterIP: 100.64.0.10 + clusterIP: fd00:5e4f:ce::a ports: - name: dns port: 53 diff --git a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content index b644aa9a39..29381b51e5 100644 --- a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content +++ b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content @@ -6,7 +6,7 @@ APIServerConfig: - kubernetes.svc.default apiServerCount: 1 authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 + bindAddress: '::' cloudProvider: aws enableAdmissionPlugins: - NamespaceLifecycle @@ -39,7 +39,7 @@ APIServerConfig: securePort: 443 serviceAccountIssuer: https://api.internal.minimal-ipv6.example.com serviceAccountJWKSURI: https://api.internal.minimal-ipv6.example.com/openid/v1/jwks - serviceClusterIPRange: 100.64.0.0/13 + serviceClusterIPRange: fd00:5e4f:ce::/108 storageBackend: etcd3 ServiceAccountPublicKeys: | -----BEGIN RSA PUBLIC KEY----- @@ -235,7 +235,7 @@ KubeletConfig: cgroupDriver: systemd cgroupRoot: / cloudProvider: aws - clusterDNS: 100.64.0.10 + clusterDNS: fd00:5e4f:ce::a clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% @@ -249,7 +249,7 @@ KubeletConfig: node-role.kubernetes.io/control-plane: "" node-role.kubernetes.io/master: "" node.kubernetes.io/exclude-from-external-load-balancers: "" - nonMasqueradeCIDR: 100.64.0.0/10 + nonMasqueradeCIDR: ::/0 podManifestPath: /etc/kubernetes/manifests registerSchedulable: false UpdatePolicy: automatic diff --git a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_nodeupconfig-nodes_content index 0c6edad5a4..da53097d77 100644 --- a/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_nodeupconfig-nodes_content +++ b/tests/integration/update_cluster/ipv6-cloudipam/data/aws_s3_bucket_object_nodeupconfig-nodes_content @@ -42,7 +42,7 @@ KubeletConfig: cgroupDriver: systemd cgroupRoot: / cloudProvider: aws - clusterDNS: 100.64.0.10 + clusterDNS: fd00:5e4f:ce::a clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% @@ -53,7 +53,7 @@ KubeletConfig: nodeLabels: kubernetes.io/role: node node-role.kubernetes.io/node: "" - nonMasqueradeCIDR: 100.64.0.0/10 + nonMasqueradeCIDR: ::/0 podManifestPath: /etc/kubernetes/manifests UpdatePolicy: automatic channels: diff --git a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content index 8091484c79..ddb95aa401 100644 --- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content @@ -54,7 +54,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 35f45e466345bbf440198f73fe9c6ab8f87ae8ed7ab714c9930dd76a5fdd60f0 + manifestHash: 3560289593c612da551bb62ce4e04c12ff4524d0a58d90d6def8df5d05a4298e name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content index 763abb4bbf..ea9b8b07e6 100644 --- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -45,7 +45,7 @@ data: enable-endpoint-health-checking: "true" enable-ipv4: "true" enable-ipv6: "false" - enable-ipv6-masquerade: "true" + enable-ipv6-masquerade: "false" enable-l7-proxy: "true" enable-node-port: "false" enable-remote-node-identity: "true" diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content index 3d0b9d0829..d12a733f27 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content @@ -54,7 +54,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: d3bfdf14497029e5668a72dab8413a302db8899ce951a99661922c2f52af135b + manifestHash: 0b45bffaea8cbfd5a8c163753a5783501b605e3a787bba0af9562bee6a4cb52c name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content index 46f9216251..b3ad08323b 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -45,7 +45,7 @@ data: enable-endpoint-health-checking: "true" enable-ipv4: "true" enable-ipv6: "false" - enable-ipv6-masquerade: "true" + enable-ipv6-masquerade: "false" enable-l7-proxy: "true" enable-node-port: "false" enable-remote-node-identity: "true" diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index cda75860fc..9e98963fed 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -61,7 +61,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 0f2a1d439e26214d53ea5a403d87f2ef7e7168bf0048f9be8c5b7d5e1cc7d963 + manifestHash: 3508e7d209ec49e2bff9a94f205b1cb5425d3bef6c47e5ecf16877ecc8345ee9 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml index 285b6f4cef..4c6568c435 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml @@ -68,7 +68,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 0f2a1d439e26214d53ea5a403d87f2ef7e7168bf0048f9be8c5b7d5e1cc7d963 + manifestHash: 3508e7d209ec49e2bff9a94f205b1cb5425d3bef6c47e5ecf16877ecc8345ee9 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml index 64761ee451..91a1a28eb5 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml @@ -61,7 +61,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 0f2a1d439e26214d53ea5a403d87f2ef7e7168bf0048f9be8c5b7d5e1cc7d963 + manifestHash: 3508e7d209ec49e2bff9a94f205b1cb5425d3bef6c47e5ecf16877ecc8345ee9 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml index f3507b989c..25ed2b62b9 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml @@ -75,7 +75,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 0f2a1d439e26214d53ea5a403d87f2ef7e7168bf0048f9be8c5b7d5e1cc7d963 + manifestHash: 3508e7d209ec49e2bff9a94f205b1cb5425d3bef6c47e5ecf16877ecc8345ee9 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml index f05e41b8a0..b3532d15cf 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml @@ -68,7 +68,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.10.yaml - manifestHash: 0f2a1d439e26214d53ea5a403d87f2ef7e7168bf0048f9be8c5b7d5e1cc7d963 + manifestHash: 3508e7d209ec49e2bff9a94f205b1cb5425d3bef6c47e5ecf16877ecc8345ee9 name: networking.cilium.io needsRollingUpdate: all selector: