diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 10a7e4c890..9208a67615 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -299,7 +299,7 @@ func TestMinimalIPv6(t *testing.T) { // TestMinimalIPv6 runs the test on a minimum IPv6 configuration func TestMinimalIPv6Private(t *testing.T) { newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-private"). - withAddons(awsCCMAddon, awsEBSCSIAddon, dnsControllerAddon). + withAddons(awsCCMAddon, awsEBSCSIAddon, dnsControllerAddon, leaderElectionAddon). runTestTerraformAWS(t) } diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index f2f7cf3c5e..129ecaf84e 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -338,7 +338,11 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList { fieldSubnet := fieldSpec.Child("subnets").Index(i) if s.CIDR == "" { if requiresSubnetCIDR && strict { - allErrs = append(allErrs, field.Required(fieldSubnet.Child("cidr"), "subnet did not have a cidr set")) + if !strings.Contains(c.Spec.NonMasqueradeCIDR, ":") || s.IPv6CIDR == "" { + allErrs = append(allErrs, field.Required(fieldSubnet.Child("cidr"), "subnet did not have a cidr set")) + } else if c.IsKubernetesLT("1.23") { + allErrs = append(allErrs, field.Required(fieldSubnet.Child("cidr"), "IPv6-only subnets require Kubernetes 1.23+")) + } } } else { _, subnetCIDR, err := net.ParseCIDR(s.CIDR) diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index e4da64a3aa..cab4ee688f 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -104,7 +104,7 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie } if spec.Topology != nil { - allErrs = append(allErrs, validateTopology(spec.Topology, fieldPath.Child("topology"))...) + allErrs = append(allErrs, validateTopology(c, spec.Topology, fieldPath.Child("topology"))...) } // UpdatePolicy @@ -360,7 +360,7 @@ func validateIPv6CIDR(cidr string, fieldPath *field.Path) field.ErrorList { return allErrs } -func validateTopology(topology *kops.TopologySpec, fieldPath *field.Path) field.ErrorList { +func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if topology.Masters == "" { @@ -373,6 +373,10 @@ func validateTopology(topology *kops.TopologySpec, fieldPath *field.Path) field. allErrs = append(allErrs, field.Required(fieldPath.Child("nodes"), "")) } else { allErrs = append(allErrs, IsValidValue(fieldPath.Child("nodes"), &topology.Nodes, kops.SupportedTopologies)...) + + if topology.Nodes == "private" && c.Spec.IsIPv6Only() && c.IsKubernetesLT("1.23") { + allErrs = append(allErrs, field.Forbidden(fieldPath.Child("nodes"), "private topology in IPv6 clusters requires Kubernetes 1.23+")) + } } if topology.Bastion != nil { diff --git a/pkg/model/awsmodel/network.go b/pkg/model/awsmodel/network.go index 74685b288b..7bcc61ed32 100644 --- a/pkg/model/awsmodel/network.go +++ b/pkg/model/awsmodel/network.go @@ -254,7 +254,6 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { Lifecycle: b.Lifecycle, VPC: b.LinkToVPC(), AvailabilityZone: fi.String(subnetSpec.Zone), - CIDR: fi.String(subnetSpec.CIDR), Shared: fi.Bool(sharedSubnet), Tags: tags, } @@ -263,6 +262,10 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { subnet.ResourceBasedNaming = fi.Bool(true) } + if subnetSpec.CIDR != "" { + subnet.CIDR = fi.String(subnetSpec.CIDR) + } + if subnetSpec.IPv6CIDR != "" { if !sharedVPC { subnet.AmazonIPv6CIDR = b.LinkToAmazonVPCIPv6CIDR() diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data index 2ec83c60ff..e10bb82d26 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data @@ -134,15 +134,15 @@ cloudConfig: containerRuntime: containerd containerd: logLevel: info - version: 1.4.12 + version: 1.6.0-beta.4 docker: skipInstall: true encryptionConfig: null etcdClusters: events: - version: 3.4.13 + version: 3.5.1 main: - version: 3.4.13 + version: 3.5.1 kubeAPIServer: allowPrivileged: true anonymousAuth: false @@ -166,7 +166,7 @@ kubeAPIServer: - https://127.0.0.1:4001 etcdServersOverrides: - /events#https://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.21.0 + image: k8s.gcr.io/kube-apiserver:v1.23.0 kubeletPreferredAddressTypes: - InternalIP - Hostname @@ -196,17 +196,17 @@ kubeControllerManager: - -nodeipam featureGates: CSIMigrationAWS: "true" - image: k8s.gcr.io/kube-controller-manager:v1.21.0 + image: k8s.gcr.io/kube-controller-manager:v1.23.0 leaderElection: leaderElect: true logLevel: 2 useServiceAccountCredentials: true kubeProxy: cpuRequest: 100m - image: k8s.gcr.io/kube-proxy:v1.21.0 + image: k8s.gcr.io/kube-proxy:v1.23.0 logLevel: 2 kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.21.0 + image: k8s.gcr.io/kube-scheduler:v1.23.0 leaderElection: leaderElect: true logLevel: 2 @@ -226,6 +226,7 @@ kubelet: logLevel: 2 networkPluginName: cni podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true masterKubelet: anonymousAuth: false cgroupDriver: systemd @@ -242,6 +243,7 @@ masterKubelet: logLevel: 2 networkPluginName: cni podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true registerSchedulable: false __EOF_CLUSTER_SPEC @@ -251,7 +253,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal-ipv6.example.com InstanceGroupName: master-us-test-1a InstanceGroupRole: Master -NodeupConfigHash: 125RlQGI9pGb/KE6nPLwc48XGm9VQ4cnTWB2ijOHGtw= +NodeupConfigHash: lhBGIbUgHvtWo0RjBS5OMF+gfW20GroKLr1lB9zTiK4= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data index 622fd91039..b561b701d4 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data @@ -134,12 +134,12 @@ cloudConfig: containerRuntime: containerd containerd: logLevel: info - version: 1.4.12 + version: 1.6.0-beta.4 docker: skipInstall: true kubeProxy: cpuRequest: 100m - image: k8s.gcr.io/kube-proxy:v1.21.0 + image: k8s.gcr.io/kube-proxy:v1.23.0 logLevel: 2 kubelet: anonymousAuth: false @@ -157,6 +157,7 @@ kubelet: logLevel: 2 networkPluginName: cni podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true __EOF_CLUSTER_SPEC @@ -165,7 +166,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal-ipv6.example.com InstanceGroupName: nodes InstanceGroupRole: Node -NodeupConfigHash: Pyc1ooFcHOEr89bybwfSkbPNXGlCaXmLybVqMYEbaZo= +NodeupConfigHash: /2OlsRVRArRL6ffpG51ysZBDm3bfGrJv38ryXueFgMk= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_cluster-completed.spec_content index 4fa57936ee..989563d343 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_cluster-completed.spec_content @@ -24,7 +24,7 @@ spec: clusterCIDR: ::/0 clusterName: minimal-ipv6.example.com configureCloudRoutes: false - image: k8s.gcr.io/provider-aws/cloud-controller-manager:v1.21.0-alpha.0 + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:latest cloudProvider: aws clusterDNSDomain: cluster.local configBase: memfs://clusters.example.com/minimal-ipv6.example.com @@ -32,7 +32,7 @@ spec: containerRuntime: containerd containerd: logLevel: info - version: 1.4.12 + version: 1.6.0-beta.4 dnsZone: Z1AFAKE1ZON3YO docker: skipInstall: true @@ -43,14 +43,14 @@ spec: - instanceGroup: master-us-test-1a name: us-test-1a name: main - version: 3.4.13 + version: 3.5.1 - backups: backupStore: memfs://clusters.example.com/minimal-ipv6.example.com/backups/etcd/events etcdMembers: - instanceGroup: master-us-test-1a name: us-test-1a name: events - version: 3.4.13 + version: 3.5.1 externalDns: provider: dns-controller iam: @@ -79,7 +79,7 @@ spec: - https://127.0.0.1:4001 etcdServersOverrides: - /events#https://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.21.0 + image: k8s.gcr.io/kube-apiserver:v1.23.0 kubeletPreferredAddressTypes: - InternalIP - Hostname @@ -109,7 +109,7 @@ spec: - -nodeipam featureGates: CSIMigrationAWS: "true" - image: k8s.gcr.io/kube-controller-manager:v1.21.0 + image: k8s.gcr.io/kube-controller-manager:v1.23.0 leaderElection: leaderElect: true logLevel: 2 @@ -132,10 +132,10 @@ spec: - fd00:ec2::253 kubeProxy: cpuRequest: 100m - image: k8s.gcr.io/kube-proxy:v1.21.0 + image: k8s.gcr.io/kube-proxy:v1.23.0 logLevel: 2 kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.21.0 + image: k8s.gcr.io/kube-scheduler:v1.23.0 leaderElection: leaderElect: true logLevel: 2 @@ -155,10 +155,11 @@ spec: logLevel: 2 networkPluginName: cni podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true kubernetesApiAccess: - 0.0.0.0/0 - ::/0 - kubernetesVersion: 1.21.0 + kubernetesVersion: 1.23.0 masterInternalName: api.internal.minimal-ipv6.example.com masterKubelet: anonymousAuth: false @@ -176,6 +177,7 @@ spec: logLevel: 2 networkPluginName: cni podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true registerSchedulable: false masterPublicName: api.minimal-ipv6.example.com networkCIDR: 172.20.0.0/16 diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_etcd-cluster-spec-events_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_etcd-cluster-spec-events_content index bb8ddb0e2e..cba4cf5b2d 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_etcd-cluster-spec-events_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_etcd-cluster-spec-events_content @@ -1,4 +1,4 @@ { "memberCount": 1, - "etcdVersion": "3.4.13" + "etcdVersion": "3.5.1" } diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_etcd-cluster-spec-main_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_etcd-cluster-spec-main_content index bb8ddb0e2e..cba4cf5b2d 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_etcd-cluster-spec-main_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_etcd-cluster-spec-main_content @@ -1,4 +1,4 @@ { "memberCount": 1, - "etcdVersion": "3.4.13" + "etcdVersion": "3.5.1" } diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content index abc8a8a96c..a41e242b95 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content @@ -31,7 +31,7 @@ spec: env: - name: KUBERNETES_SERVICE_HOST value: 127.0.0.1 - image: k8s.gcr.io/provider-aws/cloud-controller-manager:v1.21.0-alpha.0 + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:latest imagePullPolicy: IfNotPresent name: aws-cloud-controller-manager resources: diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content index 6b3cec67a7..b7e3c3dfb0 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 37b1ce4604ef11ae982a5d6f67af2f7865a9a75280cdf21911274a50a2438ed0 + manifestHash: f11dedb11c9ee1f3cc9edef8f3c2c59e84e8a3b81acef0e4b99236d9fb67b54c name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: @@ -32,6 +32,13 @@ spec: selector: k8s-addon: kubelet-api.rbac.addons.k8s.io version: 9.99.0 + - id: k8s-1.23 + manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml + manifestHash: 6211e71f8175cebcba7812f74c41d175604cbff7bab9ac788f80bac290a7b981 + name: leader-migration.rbac.addons.k8s.io + selector: + k8s-addon: leader-migration.rbac.addons.k8s.io + version: 9.99.0 - manifest: limit-range.addons.k8s.io/v1.5.0.yaml manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2 name: limit-range.addons.k8s.io @@ -54,7 +61,7 @@ spec: version: 9.99.0 - id: k8s-1.18 manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml - manifestHash: 2177fcb1b508f6e280e888dd76cc8ac82847df6c25781e7ee90d563955a3cd9d + manifestHash: ad120019d9cfca34c49d8fdc1b8a8da58cc8b68c0e18edf7a367ee4d7147973d name: aws-cloud-controller.addons.k8s.io selector: k8s-addon: aws-cloud-controller.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content index 25ad4ab8f0..91af3a5eb7 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content @@ -1,7 +1,7 @@ apiVersion: v1 data: config.yaml: | - {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal-ipv6.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal-ipv6.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]},"enableCloudIPAM":true} + {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal-ipv6.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal-ipv6.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true},"enableCloudIPAM":true} kind: ConfigMap metadata: creationTimestamp: null diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content new file mode 100644 index 0000000000..0005b049b5 --- /dev/null +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content @@ -0,0 +1,49 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +rules: +- apiGroups: + - coordination.k8s.io + resourceNames: + - cloud-provider-extraction-migration + resources: + - leases + verbs: + - create + - list + - get + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system::leader-locking-migration +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:kube-controller-manager +- kind: ServiceAccount + name: kube-controller-manager + namespace: kube-system +- kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content index f87cbbaf8b..360447f07c 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content @@ -22,7 +22,7 @@ APIServerConfig: - https://127.0.0.1:4001 etcdServersOverrides: - /events#https://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.21.0 + image: k8s.gcr.io/kube-apiserver:v1.23.0 kubeletPreferredAddressTypes: - InternalIP - Hostname @@ -52,17 +52,17 @@ APIServerConfig: -----END RSA PUBLIC KEY----- Assets: amd64: - - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet - - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl - - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz - - f6120552408175ca332fd3b5d31c5edd115d8426d6731664e4ea3951c5eee3b4@https://github.com/containerd/containerd/releases/download/v1.4.12/cri-containerd-cni-1.4.12-linux-amd64.tar.gz + - 4756ff345dd80704b749d87efb8eb294a143a1f4a251ec586197d26ad20ea518@https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubelet + - 2d0f5ba6faa787878b642c151ccb2c3390ce4c1e6c8e2b59568b3869ba407c4f@https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - 9a33654970a73bec7ae32b1e7dffd4dd0217291ca2072f6944e241ee1b1d7fe5@https://github.com/containerd/containerd/releases/download/v1.6.0-beta.4/cri-containerd-cni-1.6.0-beta.4-linux-amd64.tar.gz - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 arm64: - - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet - - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl - - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - - 87a4219c54552797ffd38790b72832372a90eceb7c8e451c36a682093d57dae6@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.11.tgz + - a546fb7ccce69c4163e4a0b19a31f30ea039b4e4560c23fd6e3016e2b2dfd0d9@https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/arm64/kubelet + - 1d77d6027fc8dfed772609ad9bd68f611b7e4ce73afa949f27084ad3a92b15fe@https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - c84a7600a5d95306be48c85a5966011f714ee78ba990ee4526b25f042d17528d@https://github.com/containerd/containerd/releases/download/v1.6.0-beta.4/cri-containerd-cni-1.6.0-beta.4-linux-arm64.tar.gz - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 CAs: @@ -252,16 +252,18 @@ KubeletConfig: node-role.kubernetes.io/master: "" node.kubernetes.io/exclude-from-external-load-balancers: "" podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true registerSchedulable: false UpdatePolicy: automatic channels: - memfs://clusters.example.com/minimal-ipv6.example.com/addons/bootstrap-channel.yaml containerdConfig: logLevel: info - version: 1.4.12 + version: 1.6.0-beta.4 etcdManifests: - memfs://clusters.example.com/minimal-ipv6.example.com/manifests/etcd/main.yaml - memfs://clusters.example.com/minimal-ipv6.example.com/manifests/etcd/events.yaml staticManifests: - key: kube-apiserver-healthcheck path: manifests/static/kube-apiserver-healthcheck.yaml +useInstanceIDForNodeName: true diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_nodeupconfig-nodes_content index f36119f076..b893501909 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_nodeupconfig-nodes_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_nodeupconfig-nodes_content @@ -1,14 +1,14 @@ Assets: amd64: - - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet - - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl - - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz - - f6120552408175ca332fd3b5d31c5edd115d8426d6731664e4ea3951c5eee3b4@https://github.com/containerd/containerd/releases/download/v1.4.12/cri-containerd-cni-1.4.12-linux-amd64.tar.gz + - 4756ff345dd80704b749d87efb8eb294a143a1f4a251ec586197d26ad20ea518@https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubelet + - 2d0f5ba6faa787878b642c151ccb2c3390ce4c1e6c8e2b59568b3869ba407c4f@https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - 9a33654970a73bec7ae32b1e7dffd4dd0217291ca2072f6944e241ee1b1d7fe5@https://github.com/containerd/containerd/releases/download/v1.6.0-beta.4/cri-containerd-cni-1.6.0-beta.4-linux-amd64.tar.gz arm64: - - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet - - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl - - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - - 87a4219c54552797ffd38790b72832372a90eceb7c8e451c36a682093d57dae6@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.11.tgz + - a546fb7ccce69c4163e4a0b19a31f30ea039b4e4560c23fd6e3016e2b2dfd0d9@https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/arm64/kubelet + - 1d77d6027fc8dfed772609ad9bd68f611b7e4ce73afa949f27084ad3a92b15fe@https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - c84a7600a5d95306be48c85a5966011f714ee78ba990ee4526b25f042d17528d@https://github.com/containerd/containerd/releases/download/v1.6.0-beta.4/cri-containerd-cni-1.6.0-beta.4-linux-arm64.tar.gz CAs: kubernetes-ca: | -----BEGIN CERTIFICATE----- @@ -56,9 +56,11 @@ KubeletConfig: kubernetes.io/role: node node-role.kubernetes.io/node: "" podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true UpdatePolicy: automatic channels: - memfs://clusters.example.com/minimal-ipv6.example.com/addons/bootstrap-channel.yaml containerdConfig: logLevel: info - version: 1.4.12 + version: 1.6.0-beta.4 +useInstanceIDForNodeName: true diff --git a/tests/integration/update_cluster/minimal-ipv6-private/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal-ipv6-private/in-v1alpha2.yaml index dfe5d324c7..f0376c7618 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/minimal-ipv6-private/in-v1alpha2.yaml @@ -33,7 +33,7 @@ spec: iam: {} kubelet: anonymousAuth: false - kubernetesVersion: v1.21.0 + kubernetesVersion: v1.23.0 masterInternalName: api.internal.minimal-ipv6.example.com masterPublicName: api.minimal-ipv6.example.com networkCIDR: 172.20.0.0/16 diff --git a/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf b/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf index d46aaae1df..5e9479be13 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-ipv6-private/kubernetes.tf @@ -818,6 +818,14 @@ resource "aws_s3_bucket_object" "minimal-ipv6-example-com-addons-kubelet-api-rba server_side_encryption = "AES256" } +resource "aws_s3_bucket_object" "minimal-ipv6-example-com-addons-leader-migration-rbac-addons-k8s-io-k8s-1-23" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content") + key = "clusters.example.com/minimal-ipv6.example.com/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_bucket_object" "minimal-ipv6-example-com-addons-limit-range-addons-k8s-io" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-limit-range.addons.k8s.io_content") diff --git a/upup/pkg/fi/cloudup/awstasks/subnet.go b/upup/pkg/fi/cloudup/awstasks/subnet.go index 000769a363..2c1f7a73ca 100644 --- a/upup/pkg/fi/cloudup/awstasks/subnet.go +++ b/upup/pkg/fi/cloudup/awstasks/subnet.go @@ -106,7 +106,7 @@ func (e *Subnet) Find(c *fi.Context) (*Subnet, error) { actual.ResourceBasedNaming = fi.Bool(aws.StringValue(subnet.PrivateDnsNameOptionsOnLaunch.HostnameType) == ec2.HostnameTypeResourceName) if *actual.ResourceBasedNaming { - if !aws.BoolValue(subnet.PrivateDnsNameOptionsOnLaunch.EnableResourceNameDnsARecord) { + if fi.StringValue(actual.CIDR) != "" && !aws.BoolValue(subnet.PrivateDnsNameOptionsOnLaunch.EnableResourceNameDnsARecord) { actual.ResourceBasedNaming = nil } if fi.StringValue(actual.IPv6CIDR) != "" && !aws.BoolValue(subnet.PrivateDnsNameOptionsOnLaunch.EnableResourceNameDnsAAAARecord) { @@ -175,9 +175,8 @@ func (s *Subnet) CheckChanges(a, e, changes *Subnet) error { errors = append(errors, field.Required(fieldPath.Child("VPC"), "must specify a VPC")) } - if e.CIDR == nil { - // TODO: Auto-assign CIDR? - errors = append(errors, field.Required(fieldPath.Child("CIDR"), "must specify a CIDR")) + if e.CIDR == nil && e.IPv6CIDR == nil { + errors = append(errors, field.Required(fieldPath.Child("CIDR"), "must specify a CIDR or IPv6CIDR")) } } @@ -256,7 +255,7 @@ func (_ *Subnet) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Subnet) error { } if a == nil { - klog.V(2).Infof("Creating Subnet with CIDR: %q", *e.CIDR) + klog.V(2).Infof("Creating Subnet with CIDR: %q IPv6CIDR: %q", fi.StringValue(e.CIDR), fi.StringValue(e.IPv6CIDR)) request := &ec2.CreateSubnetInput{ CidrBlock: e.CIDR, @@ -266,6 +265,10 @@ func (_ *Subnet) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Subnet) error { TagSpecifications: awsup.EC2TagSpecification(ec2.ResourceTypeSubnet, e.Tags), } + if e.CIDR == nil { + request.Ipv6Native = aws.Bool(true) + } + response, err := t.Cloud.EC2().CreateSubnet(request) if err != nil { return fmt.Errorf("error creating subnet: %v", err) @@ -300,13 +303,24 @@ func (_ *Subnet) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Subnet) error { return fmt.Errorf("error modifying hostname type: %w", err) } - request = &ec2.ModifySubnetAttributeInput{ - SubnetId: e.ID, - EnableResourceNameDnsARecordOnLaunch: &ec2.AttributeBooleanValue{Value: changes.ResourceBasedNaming}, - } - _, err = t.Cloud.EC2().ModifySubnetAttribute(request) - if err != nil { - return fmt.Errorf("error modifying A records: %w", err) + if fi.StringValue(e.CIDR) == "" { + request = &ec2.ModifySubnetAttributeInput{ + SubnetId: e.ID, + EnableDns64: &ec2.AttributeBooleanValue{Value: aws.Bool(true)}, + } + _, err = t.Cloud.EC2().ModifySubnetAttribute(request) + if err != nil { + return fmt.Errorf("error enabling DNS64: %w", err) + } + } else { + request = &ec2.ModifySubnetAttributeInput{ + SubnetId: e.ID, + EnableResourceNameDnsARecordOnLaunch: &ec2.AttributeBooleanValue{Value: changes.ResourceBasedNaming}, + } + _, err = t.Cloud.EC2().ModifySubnetAttribute(request) + if err != nil { + return fmt.Errorf("error modifying A records: %w", err) + } } if fi.StringValue(e.IPv6CIDR) != "" { diff --git a/upup/pkg/fi/cloudup/subnets.go b/upup/pkg/fi/cloudup/subnets.go index b3addefb6f..a16b2571a2 100644 --- a/upup/pkg/fi/cloudup/subnets.go +++ b/upup/pkg/fi/cloudup/subnets.go @@ -174,6 +174,10 @@ func assignCIDRsToSubnets(c *kops.Cluster, cloud fi.Cloud) error { if subnet.CIDR != "" { continue } + // TODO: Replace this with a check against type "dualstack" if has IPv6CIDR + if subnet.IPv6CIDR != "" && subnet.Name != subnet.Zone { + continue + } if len(bigCIDRs) == 0 { return fmt.Errorf("insufficient (big) CIDRs remaining for automatic CIDR allocation to subnet %q", subnet.Name)