From dde08e839d1dd1133058f72502cf5dec5a3ddfa2 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Sat, 11 Sep 2021 09:01:14 +0300 Subject: [PATCH] Make AWS CCM NodeIPFamilies configurable --- k8s/crds/kops.k8s.io_clusters.yaml | 6 ++ nodeup/pkg/model/BUILD.bazel | 1 + nodeup/pkg/model/cloudconfig.go | 4 +- nodeup/pkg/model/cloudconfig_test.go | 56 +++++++++++++++++++ pkg/apis/kops/componentconfig.go | 2 + pkg/apis/kops/v1alpha2/componentconfig.go | 2 + .../kops/v1alpha2/zz_generated.conversion.go | 2 + .../kops/v1alpha2/zz_generated.deepcopy.go | 5 ++ pkg/apis/kops/zz_generated.deepcopy.go | 5 ++ pkg/model/components/cloudconfiguration.go | 4 ++ 10 files changed, 85 insertions(+), 2 deletions(-) diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 4adc040e86..a0d857a655 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -343,6 +343,12 @@ spec: multizone: description: GCE cloud-config options type: boolean + nodeIPFamilies: + description: NodeIPFamilies controls the IP families reported + for each node (AWS only). + items: + type: string + type: array nodeInstancePrefix: type: string nodeTags: diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index c607459ff9..e3b5fab15c 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -107,6 +107,7 @@ go_test( "//pkg/assets:go_default_library", "//pkg/client/simple/vfsclientset:go_default_library", "//pkg/configbuilder:go_default_library", + "//pkg/diff:go_default_library", "//pkg/flagbuilder:go_default_library", "//pkg/pki:go_default_library", "//pkg/testutils:go_default_library", diff --git a/nodeup/pkg/model/cloudconfig.go b/nodeup/pkg/model/cloudconfig.go index 27bcc660b4..87e47921fa 100644 --- a/nodeup/pkg/model/cloudconfig.go +++ b/nodeup/pkg/model/cloudconfig.go @@ -102,8 +102,8 @@ func (b *CloudConfigBuilder) Build(c *fi.ModelBuilderContext) error { if cloudConfig.ElbSecurityGroup != nil { lines = append(lines, "ElbSecurityGroup = "+*cloudConfig.ElbSecurityGroup) } - if b.Cluster.Spec.IsIPv6Only() { - lines = append(lines, "NodeIPFamilies = ipv6") + for _, family := range cloudConfig.NodeIPFamilies { + lines = append(lines, "NodeIPFamilies = "+family) } case "openstack": osc := cloudConfig.Openstack diff --git a/nodeup/pkg/model/cloudconfig_test.go b/nodeup/pkg/model/cloudconfig_test.go index 8e4df1b2fe..60ed08b8d8 100644 --- a/nodeup/pkg/model/cloudconfig_test.go +++ b/nodeup/pkg/model/cloudconfig_test.go @@ -24,6 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/diff" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" ) @@ -110,3 +111,58 @@ func TestBuildAzure(t *testing.T) { t.Errorf("expected %+v, but got %+v", expected, actual) } } + +func TestBuildAWSCustomNodeIPFamilies(t *testing.T) { + cluster := &kops.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testcluster.test.com", + }, + Spec: kops.ClusterSpec{ + CloudProvider: string(kops.CloudProviderAWS), + CloudConfig: &kops.CloudConfiguration{ + NodeIPFamilies: []string{"ipv6"}, + }, + ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{ + CloudProvider: string(kops.CloudProviderAWS), + }, + NonMasqueradeCIDR: "fd00:10:96::/64", + }, + } + + b := &CloudConfigBuilder{ + NodeupModelContext: &NodeupModelContext{ + Cluster: cluster, + }, + } + ctx := &fi.ModelBuilderContext{ + Tasks: map[string]fi.Task{}, + } + if err := b.Build(ctx); err != nil { + t.Fatalf("unexpected error: %s", err) + } + var task *nodetasks.File + for _, v := range ctx.Tasks { + if f, ok := v.(*nodetasks.File); ok { + task = f + break + } + } + if task == nil { + t.Errorf("no File task found") + } + r, err := task.Contents.Open() + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + awsCloudConfig, err := ioutil.ReadAll(r) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + actual := string(awsCloudConfig) + expected := "[global]\nNodeIPFamilies = ipv6\n" + if actual != expected { + diffString := diff.FormatDiff(expected, actual) + t.Errorf("actual did not match expected:\n%s\n", diffString) + } +} diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 7c5662e380..d1ca68951f 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -839,6 +839,8 @@ type CloudConfiguration struct { Multizone *bool `json:"multizone,omitempty"` NodeTags *string `json:"nodeTags,omitempty"` NodeInstancePrefix *string `json:"nodeInstancePrefix,omitempty"` + // NodeIPFamilies controls the IP families reported for each node (AWS only). + NodeIPFamilies []string `json:"nodeIPFamilies,omitempty"` // GCEServiceAccount specifies the service account with which the GCE VM runs GCEServiceAccount string `json:"gceServiceAccount,omitempty"` // AWS cloud-config options diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index a7bf3c1fe5..103fd50b9b 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -838,6 +838,8 @@ type CloudConfiguration struct { Multizone *bool `json:"multizone,omitempty"` NodeTags *string `json:"nodeTags,omitempty"` NodeInstancePrefix *string `json:"nodeInstancePrefix,omitempty"` + // NodeIPFamilies controls the IP families reported for each node (AWS only). + NodeIPFamilies []string `json:"nodeIPFamilies,omitempty"` // GCEServiceAccount specifies the service account with which the GCE VM runs GCEServiceAccount string `json:"gceServiceAccount,omitempty"` // AWS cloud-config options diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 29b747d939..fbb2ec3938 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2021,6 +2021,7 @@ func autoConvert_v1alpha2_CloudConfiguration_To_kops_CloudConfiguration(in *Clou out.Multizone = in.Multizone out.NodeTags = in.NodeTags out.NodeInstancePrefix = in.NodeInstancePrefix + out.NodeIPFamilies = in.NodeIPFamilies out.GCEServiceAccount = in.GCEServiceAccount out.DisableSecurityGroupIngress = in.DisableSecurityGroupIngress out.ElbSecurityGroup = in.ElbSecurityGroup @@ -2073,6 +2074,7 @@ func autoConvert_kops_CloudConfiguration_To_v1alpha2_CloudConfiguration(in *kops out.Multizone = in.Multizone out.NodeTags = in.NodeTags out.NodeInstancePrefix = in.NodeInstancePrefix + out.NodeIPFamilies = in.NodeIPFamilies out.GCEServiceAccount = in.GCEServiceAccount out.DisableSecurityGroupIngress = in.DisableSecurityGroupIngress out.ElbSecurityGroup = in.ElbSecurityGroup diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index c195440dd6..b3ee5b13c3 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -624,6 +624,11 @@ func (in *CloudConfiguration) DeepCopyInto(out *CloudConfiguration) { *out = new(string) **out = **in } + if in.NodeIPFamilies != nil { + in, out := &in.NodeIPFamilies, &out.NodeIPFamilies + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.DisableSecurityGroupIngress != nil { in, out := &in.DisableSecurityGroupIngress, &out.DisableSecurityGroupIngress *out = new(bool) diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 65a38fef44..048e8c3e78 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -708,6 +708,11 @@ func (in *CloudConfiguration) DeepCopyInto(out *CloudConfiguration) { *out = new(string) **out = **in } + if in.NodeIPFamilies != nil { + in, out := &in.NodeIPFamilies, &out.NodeIPFamilies + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.DisableSecurityGroupIngress != nil { in, out := &in.DisableSecurityGroupIngress, &out.DisableSecurityGroupIngress *out = new(bool) diff --git a/pkg/model/components/cloudconfiguration.go b/pkg/model/components/cloudconfiguration.go index c8510299c6..566395f58b 100644 --- a/pkg/model/components/cloudconfiguration.go +++ b/pkg/model/components/cloudconfiguration.go @@ -52,5 +52,9 @@ func (b *CloudConfigurationOptionsBuilder) BuildOptions(o interface{}) error { c.ManageStorageClasses = manage } + if clusterSpec.IsIPv6Only() && len(c.NodeIPFamilies) == 0 { + c.NodeIPFamilies = []string{"ipv6", "ipv4"} + } + return nil }