From dd665fbe1ea0e375492ae7a11f164ad2df9b55dc Mon Sep 17 00:00:00 2001 From: shapirus Date: Wed, 25 Oct 2023 12:27:49 +0300 Subject: [PATCH] Support setting cluster-autoscaler's skip-nodes-with-custom-controller-pods parameter --- docs/addons.md | 1 + k8s/crds/kops.k8s.io_clusters.yaml | 5 +++++ pkg/apis/kops/componentconfig.go | 3 +++ pkg/apis/kops/v1alpha2/componentconfig.go | 3 +++ pkg/apis/kops/v1alpha2/zz_generated.conversion.go | 2 ++ pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go | 5 +++++ pkg/apis/kops/v1alpha3/componentconfig.go | 3 +++ pkg/apis/kops/v1alpha3/zz_generated.conversion.go | 2 ++ pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go | 5 +++++ pkg/apis/kops/zz_generated.deepcopy.go | 5 +++++ pkg/model/components/clusterautoscaler.go | 3 +++ .../data/aws_s3_object_cluster-completed.spec_content | 1 + .../data/aws_s3_object_cluster-completed.spec_content | 1 + .../data/aws_s3_object_cluster-completed.spec_content | 1 + .../data/aws_s3_object_cluster-completed.spec_content | 1 + .../data/aws_s3_object_cluster-completed.spec_content | 1 + .../data/aws_s3_object_cluster-completed.spec_content | 1 + .../data/aws_s3_object_cluster-completed.spec_content | 1 + .../data/aws_s3_object_cluster-completed.spec_content | 1 + .../data/aws_s3_object_cluster-completed.spec_content | 1 + .../cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template | 3 +++ 21 files changed, 49 insertions(+) diff --git a/docs/addons.md b/docs/addons.md index 5ca18c5c40..4b88c5ac64 100644 --- a/docs/addons.md +++ b/docs/addons.md @@ -73,6 +73,7 @@ spec: balanceSimilarNodeGroups: false awsUseStaticInstanceList: false scaleDownUtilizationThreshold: 0.5 + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true newPodScaleUpDelay: 0s diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 9ff1824b2f..d6907e3f22 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -780,6 +780,11 @@ spec: description: 'ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down. Default: 0.5' type: string + skipNodesWithCustomControllerPods: + description: 'SkipNodesWithCustomControllerPods makes the cluster + autoscaler skip scale-down of nodes with pods owned by custom + controllers. Default: true' + type: boolean skipNodesWithLocalStorage: description: 'SkipNodesWithLocalStorage makes the cluster autoscaler skip scale-down of nodes with local storage. Default: true' diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index e575b4ead0..70d8dc1258 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -1028,6 +1028,9 @@ type ClusterAutoscalerConfig struct { // ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down. // Default: 0.5 ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"` + // SkipNodesWithCustomControllerPods makes the cluster autoscaler skip scale-down of nodes with pods owned by custom controllers. + // Default: true + SkipNodesWithCustomControllerPods *bool `json:"skipNodesWithCustomControllerPods,omitempty"` // SkipNodesWithSystemPods makes the cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace. // Default: true SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index d6a2c7541e..d38c1d59b0 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -1091,6 +1091,9 @@ type ClusterAutoscalerConfig struct { // ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down. // Default: 0.5 ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"` + // SkipNodesWithCustomControllerPods makes the cluster autoscaler skip scale-down of nodes with pods owned by custom controllers. + // Default: true + SkipNodesWithCustomControllerPods *bool `json:"skipNodesWithCustomControllerPods,omitempty"` // SkipNodesWithSystemPods makes the cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace. // Default: true SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 876799c9fd..afe0773329 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2308,6 +2308,7 @@ func autoConvert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfi out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold + out.SkipNodesWithCustomControllerPods = in.SkipNodesWithCustomControllerPods out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage out.NewPodScaleUpDelay = in.NewPodScaleUpDelay @@ -2337,6 +2338,7 @@ func autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfi out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold + out.SkipNodesWithCustomControllerPods = in.SkipNodesWithCustomControllerPods out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage out.NewPodScaleUpDelay = in.NewPodScaleUpDelay diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index f43c4a81be..d19872f5a2 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -938,6 +938,11 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) { *out = new(string) **out = **in } + if in.SkipNodesWithCustomControllerPods != nil { + in, out := &in.SkipNodesWithCustomControllerPods, &out.SkipNodesWithCustomControllerPods + *out = new(bool) + **out = **in + } if in.SkipNodesWithSystemPods != nil { in, out := &in.SkipNodesWithSystemPods, &out.SkipNodesWithSystemPods *out = new(bool) diff --git a/pkg/apis/kops/v1alpha3/componentconfig.go b/pkg/apis/kops/v1alpha3/componentconfig.go index ded94c0bb8..5c889bc39c 100644 --- a/pkg/apis/kops/v1alpha3/componentconfig.go +++ b/pkg/apis/kops/v1alpha3/componentconfig.go @@ -1020,6 +1020,9 @@ type ClusterAutoscalerConfig struct { // ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down. // Default: 0.5 ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"` + // SkipNodesWithCustomControllerPods makes the cluster autoscaler skip scale-down of nodes with pods owned by custom controllers. + // Default: true + SkipNodesWithCustomControllerPods *bool `json:"skipNodesWithCustomControllerPods,omitempty"` // SkipNodesWithSystemPods makes the cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace. // Default: true SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"` diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index 023e70f969..1455573643 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -2554,6 +2554,7 @@ func autoConvert_v1alpha3_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfi out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold + out.SkipNodesWithCustomControllerPods = in.SkipNodesWithCustomControllerPods out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage out.NewPodScaleUpDelay = in.NewPodScaleUpDelay @@ -2583,6 +2584,7 @@ func autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha3_ClusterAutoscalerConfi out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold + out.SkipNodesWithCustomControllerPods = in.SkipNodesWithCustomControllerPods out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage out.NewPodScaleUpDelay = in.NewPodScaleUpDelay diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index b3e568739e..d7721968ef 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -900,6 +900,11 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) { *out = new(string) **out = **in } + if in.SkipNodesWithCustomControllerPods != nil { + in, out := &in.SkipNodesWithCustomControllerPods, &out.SkipNodesWithCustomControllerPods + *out = new(bool) + **out = **in + } if in.SkipNodesWithSystemPods != nil { in, out := &in.SkipNodesWithSystemPods, &out.SkipNodesWithSystemPods *out = new(bool) diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 7a00116824..e4a8be6726 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -997,6 +997,11 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) { *out = new(string) **out = **in } + if in.SkipNodesWithCustomControllerPods != nil { + in, out := &in.SkipNodesWithCustomControllerPods, &out.SkipNodesWithCustomControllerPods + *out = new(bool) + **out = **in + } if in.SkipNodesWithSystemPods != nil { in, out := &in.SkipNodesWithSystemPods, &out.SkipNodesWithSystemPods *out = new(bool) diff --git a/pkg/model/components/clusterautoscaler.go b/pkg/model/components/clusterautoscaler.go index 50eaca1e89..6bfda87873 100644 --- a/pkg/model/components/clusterautoscaler.go +++ b/pkg/model/components/clusterautoscaler.go @@ -69,6 +69,9 @@ func (b *ClusterAutoscalerOptionsBuilder) BuildOptions(o interface{}) error { if cas.ScaleDownUtilizationThreshold == nil { cas.ScaleDownUtilizationThreshold = fi.PtrTo("0.5") } + if cas.SkipNodesWithCustomControllerPods == nil { + cas.SkipNodesWithCustomControllerPods = fi.PtrTo(true) + } if cas.SkipNodesWithLocalStorage == nil { cas.SkipNodesWithLocalStorage = fi.PtrTo(true) } diff --git a/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content index 5b29fc84e4..a5459c6e1b 100644 --- a/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content @@ -44,6 +44,7 @@ spec: scaleDownUnneededTime: 10m0s scaleDownUnreadyTime: 20m0s scaleDownUtilizationThreshold: "0.5" + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true clusterDNSDomain: cluster.local diff --git a/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content index d1ec23e742..515f70efe7 100644 --- a/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content @@ -37,6 +37,7 @@ spec: scaleDownUnneededTime: 10m0s scaleDownUnreadyTime: 20m0s scaleDownUtilizationThreshold: "0.5" + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true clusterDNSDomain: cluster.local diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content index b207975790..24f0345290 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content @@ -40,6 +40,7 @@ spec: scaleDownUnneededTime: 10m0s scaleDownUnreadyTime: 20m0s scaleDownUtilizationThreshold: "0.5" + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true clusterDNSDomain: cluster.local diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_cluster-completed.spec_content index 87e7a324d8..e9641fad06 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_cluster-completed.spec_content @@ -41,6 +41,7 @@ spec: scaleDownUnneededTime: 10m0s scaleDownUnreadyTime: 20m0s scaleDownUtilizationThreshold: "0.5" + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true clusterDNSDomain: cluster.local diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content index 7498da92ab..a6f18e84d6 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content @@ -40,6 +40,7 @@ spec: scaleDownUnneededTime: 10m0s scaleDownUnreadyTime: 20m0s scaleDownUtilizationThreshold: "0.5" + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true clusterDNSDomain: cluster.local diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content index 4a26511cc1..85a3b1cc4e 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content @@ -40,6 +40,7 @@ spec: scaleDownUnneededTime: 10m0s scaleDownUnreadyTime: 20m0s scaleDownUtilizationThreshold: "0.5" + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true clusterDNSDomain: cluster.local diff --git a/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content index 9e8e85e998..3da37fe220 100644 --- a/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content @@ -40,6 +40,7 @@ spec: scaleDownUnneededTime: 10m0s scaleDownUnreadyTime: 20m0s scaleDownUtilizationThreshold: "0.5" + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true clusterDNSDomain: cluster.local diff --git a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content index f026082869..4a826abdf9 100644 --- a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content @@ -41,6 +41,7 @@ spec: scaleDownUnneededTime: 10m0s scaleDownUnreadyTime: 20m0s scaleDownUtilizationThreshold: "0.5" + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true clusterDNSDomain: cluster.local diff --git a/tests/integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content index 41b3470094..ddb491e1eb 100644 --- a/tests/integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content @@ -41,6 +41,7 @@ spec: scaleDownUnneededTime: 10m0s scaleDownUnreadyTime: 20m0s scaleDownUtilizationThreshold: "0.5" + skipNodesWithCustomControllerPods: true skipNodesWithLocalStorage: true skipNodesWithSystemPods: true clusterDNSDomain: cluster.local diff --git a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template index b2122d2ed1..15aa2ef120 100644 --- a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template @@ -344,6 +344,9 @@ spec: {{ end }} - --ignore-daemonsets-utilization={{ .IgnoreDaemonSetsUtilization }} - --scale-down-utilization-threshold={{ .ScaleDownUtilizationThreshold }} + {{ if IsKubernetesGTE "1.27.0" }} + - --skip-nodes-with-custom-controller-pods={{ .SkipNodesWithCustomControllerPods }} + {{ end }} - --skip-nodes-with-local-storage={{ .SkipNodesWithLocalStorage }} - --skip-nodes-with-system-pods={{ .SkipNodesWithSystemPods }} - --scale-down-delay-after-add={{ .ScaleDownDelayAfterAdd }}