IPv6 requires external CCM

cmd/kops/integration_test.go

@@ -168,6 +168,8 @@ func (i *integrationTest) withAddons(addons ...string) *integrationTest {
 }
 
 const dnsControllerAddon = "dns-controller.addons.k8s.io-k8s-1.12"
+const awsCCMAddon = "aws-cloud-controller.addons.k8s.io-k8s-1.18"
+const awsEBSCSIAddon = "aws-ebs-csi-driver.addons.k8s.io-k8s-1.17"
 
 // TestMinimal runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a
 func TestMinimal(t *testing.T) {
@@ -251,7 +253,7 @@ func TestExternalPolicies(t *testing.T) {
 // TestMinimalIPv6 runs the test on a minimum IPv6 configuration
 func TestMinimalIPv6(t *testing.T) {
 	newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6").
-		withAddons(dnsControllerAddon).
+		withAddons(awsCCMAddon, awsEBSCSIAddon, dnsControllerAddon).
 		runTestTerraformAWS(t)
 	newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6").runTestCloudformation(t)
 }
@@ -264,7 +266,7 @@ func TestMinimalIPv6Calico(t *testing.T) {
 	}
 	defer unsetFeatureFlags()
 	newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-calico").
-		withAddons(calicoAddon, dnsControllerAddon).
+		withAddons(awsCCMAddon, awsEBSCSIAddon, calicoAddon, dnsControllerAddon).
 		runTestTerraformAWS(t)
 	newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-calico").runTestCloudformation(t)
 }
@@ -277,7 +279,7 @@ func TestMinimalIPv6Cilium(t *testing.T) {
 	}
 	defer unsetFeatureFlags()
 	newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-cilium").
-		withAddons(ciliumAddon, dnsControllerAddon).
+		withAddons(awsCCMAddon, awsEBSCSIAddon, ciliumAddon, dnsControllerAddon).
 		runTestTerraformAWS(t)
 	newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-cilium").runTestCloudformation(t)
 }

pkg/apis/kops/validation/validation.go

@@ -123,6 +123,10 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
 		allErrs = append(allErrs, validateKubeAPIServer(spec.KubeAPIServer, c, fieldPath.Child("kubeAPIServer"))...)
 	}
 
+	if spec.ExternalCloudControllerManager == nil && spec.IsIPv6Only() {
+		allErrs = append(allErrs, field.Required(fieldPath.Child("cloudControllerManager"), "IPv6 requires external Cloud Controller Manager"))
+	}
+
 	if spec.KubeProxy != nil {
 		allErrs = append(allErrs, validateKubeProxy(spec.KubeProxy, fieldPath.Child("kubeProxy"))...)
 	}

tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml

@@ -9,6 +9,7 @@ spec:
   authorization:
     rbac: {}
   channel: stable
+  cloudControllerManager: {}
   cloudProvider: aws
   configBase: memfs://tests/ipv6.example.com
   etcdClusters:

tests/integration/update_cluster/minimal-ipv6-calico/in-v1alpha2.yaml

@@ -15,6 +15,10 @@ spec:
   - 0.0.0.0/0
   - ::/0
   channel: stable
+  cloudConfig:
+    awsEBSCSIDriver:
+      enabled: true
+  cloudControllerManager: {}
   cloudProvider: aws
   configBase: memfs://clusters.example.com/minimal-ipv6.example.com
   etcdClusters:

tests/integration/update_cluster/minimal-ipv6-cilium/in-v1alpha2.yaml

@@ -15,6 +15,10 @@ spec:
   - 0.0.0.0/0
   - ::/0
   channel: stable
+  cloudConfig:
+    awsEBSCSIDriver:
+      enabled: true
+  cloudControllerManager: {}
   cloudProvider: aws
   configBase: memfs://clusters.example.com/minimal-ipv6.example.com
   etcdClusters:

tests/integration/update_cluster/minimal-ipv6/in-v1alpha2.yaml

@@ -15,6 +15,10 @@ spec:
   - 0.0.0.0/0
   - ::/0
   channel: stable
+  cloudConfig:
+    awsEBSCSIDriver:
+      enabled: true
+  cloudControllerManager: {}
   cloudProvider: aws
   configBase: memfs://clusters.example.com/minimal-ipv6.example.com
   etcdClusters:

upup/pkg/fi/cloudup/new_cluster.go

@@ -969,11 +969,11 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S
 
 		if opt.IPv6 {
 			cluster.Spec.NonMasqueradeCIDR = "::/0"
+			cluster.Spec.ExternalCloudControllerManager = &api.CloudControllerManagerConfig{}
 			if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS {
 				klog.Warningf("IPv6 support is EXPERIMENTAL and can be changed or removed at any time in the future!!!")
 				for i := range cluster.Spec.Subnets {
 					// Start IPv6 CIDR numbering from "1" to reserve /64#0 for later use
-					// with NonMasqueradeCIDR, ClusterCIDR and ServiceClusterIPRange
 					cluster.Spec.Subnets[i].IPv6CIDR = fmt.Sprintf("/64#%x", i+1)
 				}
 			} else {

upup/pkg/fi/cloudup/populate_cluster_spec_test.go

@@ -87,6 +87,12 @@ func TestPopulateCluster_Subnets(t *testing.T) {
 			c.Spec.NonMasqueradeCIDR = tc.NonMasqueradeCIDR
 			c.Spec.Networking.Kubenet = nil
 			c.Spec.Networking.CNI = &kopsapi.CNINetworkingSpec{}
+			c.Spec.ExternalCloudControllerManager = &kopsapi.CloudControllerManagerConfig{}
+			c.Spec.CloudConfig = &kopsapi.CloudConfiguration{
+				AWSEBSCSIDriver: &kopsapi.AWSEBSCSIDriver{
+					Enabled: fi.Bool(true),
+				},
+			}
 
 			err := PerformAssignments(c, cloud)
 			require.NoError(t, err, "PerformAssignments")