Merge pull request #14379 from noony/cluster-autoscaler-config

cluster-autoscaler : Add scaleDownUnneededTime and scaleDownUnreadyTime

docs/addons.md

@@ -77,6 +77,8 @@ spec:
     skipNodesWithSystemPods: true
     newPodScaleUpDelay: 0s
     scaleDownDelayAfterAdd: 10m0s
+    scaleDownUnneededTime: 10m0s
+    scaleDownUnreadyTime: 20m0s
     image: <the latest supported image for the specified kubernetes version>
     cpuRequest: "100m"
     memoryRequest: "300Mi"

k8s/crds/kops.k8s.io_clusters.yaml

@@ -724,6 +724,16 @@ spec:
                     description: 'ScaleDownDelayAfterAdd determines the time after
                       scale up that scale down evaluation resumes Default: 10m0s'
                     type: string
+                  scaleDownUnneededTime:
+                    description: 'scaleDownUnneededTime determines the time a node
+                      should be unneeded before it is eligible for scale down Default:
+                      10m0s'
+                    type: string
+                  scaleDownUnreadyTime:
+                    description: 'ScaleDownUnreadyTime determines the time an unready
+                      node should be unneeded before it is eligible for scale down
+                      Default: 20m0s'
+                    type: string
                   scaleDownUtilizationThreshold:
                     description: 'ScaleDownUtilizationThreshold determines the utilization
                       threshold for node scale-down. Default: 0.5'

pkg/apis/kops/componentconfig.go

@@ -1017,6 +1017,12 @@ type ClusterAutoscalerConfig struct {
 	// ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes
 	// Default: 10m0s
 	ScaleDownDelayAfterAdd *string `json:"scaleDownDelayAfterAdd,omitempty"`
+	// scaleDownUnneededTime determines the time a node should be unneeded before it is eligible for scale down
+	// Default: 10m0s
+	ScaleDownUnneededTime *string `json:"scaleDownUnneededTime,omitempty"`
+	// ScaleDownUnreadyTime determines the time an unready node should be unneeded before it is eligible for scale down
+	// Default: 20m0s
+	ScaleDownUnreadyTime *string `json:"scaleDownUnreadyTime,omitempty"`
 	// CordonNodeBeforeTerminating should CA cordon nodes before terminating during downscale process
 	// Default: false
 	CordonNodeBeforeTerminating *bool `json:"cordonNodeBeforeTerminating,omitempty"`

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -1043,6 +1043,12 @@ type ClusterAutoscalerConfig struct {
 	// ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes
 	// Default: 10m0s
 	ScaleDownDelayAfterAdd *string `json:"scaleDownDelayAfterAdd,omitempty"`
+	// scaleDownUnneededTime determines the time a node should be unneeded before it is eligible for scale down
+	// Default: 10m0s
+	ScaleDownUnneededTime *string `json:"scaleDownUnneededTime,omitempty"`
+	// ScaleDownUnreadyTime determines the time an unready node should be unneeded before it is eligible for scale down
+	// Default: 20m0s
+	ScaleDownUnreadyTime *string `json:"scaleDownUnreadyTime,omitempty"`
 	// CordonNodeBeforeTerminating should CA cordon nodes before terminating during downscale process
 	// Default: false
 	CordonNodeBeforeTerminating *bool `json:"cordonNodeBeforeTerminating,omitempty"`

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -2334,6 +2334,8 @@ func autoConvert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfi
 	out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage
 	out.NewPodScaleUpDelay = in.NewPodScaleUpDelay
 	out.ScaleDownDelayAfterAdd = in.ScaleDownDelayAfterAdd
+	out.ScaleDownUnneededTime = in.ScaleDownUnneededTime
+	out.ScaleDownUnreadyTime = in.ScaleDownUnreadyTime
 	out.CordonNodeBeforeTerminating = in.CordonNodeBeforeTerminating
 	out.Image = in.Image
 	out.MemoryRequest = in.MemoryRequest
@@ -2358,6 +2360,8 @@ func autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfi
 	out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage
 	out.NewPodScaleUpDelay = in.NewPodScaleUpDelay
 	out.ScaleDownDelayAfterAdd = in.ScaleDownDelayAfterAdd
+	out.ScaleDownUnneededTime = in.ScaleDownUnneededTime
+	out.ScaleDownUnreadyTime = in.ScaleDownUnreadyTime
 	out.CordonNodeBeforeTerminating = in.CordonNodeBeforeTerminating
 	out.Image = in.Image
 	out.MemoryRequest = in.MemoryRequest

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -958,6 +958,16 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.ScaleDownUnneededTime != nil {
+		in, out := &in.ScaleDownUnneededTime, &out.ScaleDownUnneededTime
+		*out = new(string)
+		**out = **in
+	}
+	if in.ScaleDownUnreadyTime != nil {
+		in, out := &in.ScaleDownUnreadyTime, &out.ScaleDownUnreadyTime
+		*out = new(string)
+		**out = **in
+	}
 	if in.CordonNodeBeforeTerminating != nil {
 		in, out := &in.CordonNodeBeforeTerminating, &out.CordonNodeBeforeTerminating
 		*out = new(bool)

pkg/apis/kops/v1alpha3/componentconfig.go

@@ -1014,6 +1014,12 @@ type ClusterAutoscalerConfig struct {
 	// ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes
 	// Default: 10m0s
 	ScaleDownDelayAfterAdd *string `json:"scaleDownDelayAfterAdd,omitempty"`
+	// scaleDownUnneededTime determines the time a node should be unneeded before it is eligible for scale down
+	// Default: 10m0s
+	ScaleDownUnneededTime *string `json:"scaleDownUnneededTime,omitempty"`
+	// ScaleDownUnreadyTime determines the time an unready node should be unneeded before it is eligible for scale down
+	// Default: 20m0s
+	ScaleDownUnreadyTime *string `json:"scaleDownUnreadyTime,omitempty"`
 	// CordonNodeBeforeTerminating should CA cordon nodes before terminating during downscale process
 	// Default: false
 	CordonNodeBeforeTerminating *bool `json:"cordonNodeBeforeTerminating,omitempty"`

pkg/apis/kops/v1alpha3/zz_generated.conversion.go

@@ -2445,6 +2445,8 @@ func autoConvert_v1alpha3_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfi
 	out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage
 	out.NewPodScaleUpDelay = in.NewPodScaleUpDelay
 	out.ScaleDownDelayAfterAdd = in.ScaleDownDelayAfterAdd
+	out.ScaleDownUnneededTime = in.ScaleDownUnneededTime
+	out.ScaleDownUnreadyTime = in.ScaleDownUnreadyTime
 	out.CordonNodeBeforeTerminating = in.CordonNodeBeforeTerminating
 	out.Image = in.Image
 	out.MemoryRequest = in.MemoryRequest
@@ -2469,6 +2471,8 @@ func autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha3_ClusterAutoscalerConfi
 	out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage
 	out.NewPodScaleUpDelay = in.NewPodScaleUpDelay
 	out.ScaleDownDelayAfterAdd = in.ScaleDownDelayAfterAdd
+	out.ScaleDownUnneededTime = in.ScaleDownUnneededTime
+	out.ScaleDownUnreadyTime = in.ScaleDownUnreadyTime
 	out.CordonNodeBeforeTerminating = in.CordonNodeBeforeTerminating
 	out.Image = in.Image
 	out.MemoryRequest = in.MemoryRequest

pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go

@@ -926,6 +926,16 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.ScaleDownUnneededTime != nil {
+		in, out := &in.ScaleDownUnneededTime, &out.ScaleDownUnneededTime
+		*out = new(string)
+		**out = **in
+	}
+	if in.ScaleDownUnreadyTime != nil {
+		in, out := &in.ScaleDownUnreadyTime, &out.ScaleDownUnreadyTime
+		*out = new(string)
+		**out = **in
+	}
 	if in.CordonNodeBeforeTerminating != nil {
 		in, out := &in.CordonNodeBeforeTerminating, &out.CordonNodeBeforeTerminating
 		*out = new(bool)

pkg/apis/kops/zz_generated.deepcopy.go

@@ -1023,6 +1023,16 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.ScaleDownUnneededTime != nil {
+		in, out := &in.ScaleDownUnneededTime, &out.ScaleDownUnneededTime
+		*out = new(string)
+		**out = **in
+	}
+	if in.ScaleDownUnreadyTime != nil {
+		in, out := &in.ScaleDownUnreadyTime, &out.ScaleDownUnreadyTime
+		*out = new(string)
+		**out = **in
+	}
 	if in.CordonNodeBeforeTerminating != nil {
 		in, out := &in.CordonNodeBeforeTerminating, &out.CordonNodeBeforeTerminating
 		*out = new(bool)

pkg/model/components/clusterautoscaler.go

@@ -86,6 +86,12 @@ func (b *ClusterAutoscalerOptionsBuilder) BuildOptions(o interface{}) error {
 	if cas.ScaleDownDelayAfterAdd == nil {
 		cas.ScaleDownDelayAfterAdd = fi.String("10m0s")
 	}
+	if cas.ScaleDownUnneededTime == nil {
+		cas.ScaleDownUnneededTime = fi.String("10m0s")
+	}
+	if cas.ScaleDownUnreadyTime == nil {
+		cas.ScaleDownUnreadyTime = fi.String("20m0s")
+	}
 	if cas.MaxNodeProvisionTime == "" {
 		cas.MaxNodeProvisionTime = "15m0s"
 	}

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content

@@ -37,6 +37,8 @@ spec:
     maxNodeProvisionTime: 15m0s
     newPodScaleUpDelay: 0s
     scaleDownDelayAfterAdd: 10m0s
+    scaleDownUnneededTime: 10m0s
+    scaleDownUnreadyTime: 20m0s
     scaleDownUtilizationThreshold: "0.5"
     skipNodesWithLocalStorage: true
     skipNodesWithSystemPods: true

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 3aa20855e262fa4427110cb8c02c2062db67878695d8861589b2cbc6d8572d76
+    manifestHash: 47e4e76613b490ee644bbba65371780a4920a76756e02fc5feea4610644c2552
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -328,6 +328,8 @@ spec:
         - --skip-nodes-with-local-storage=true
         - --skip-nodes-with-system-pods=true
         - --scale-down-delay-after-add=10m0s
+        - --scale-down-unneeded-time=10m0s
+        - --scale-down-unready-time=20m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
         - --cordon-node-before-terminating=true

tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_cluster-completed.spec_content

@@ -37,6 +37,8 @@ spec:
     maxNodeProvisionTime: 15m0s
     newPodScaleUpDelay: 0s
     scaleDownDelayAfterAdd: 10m0s
+    scaleDownUnneededTime: 10m0s
+    scaleDownUnreadyTime: 20m0s
     scaleDownUtilizationThreshold: "0.5"
     skipNodesWithLocalStorage: true
     skipNodesWithSystemPods: true

tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 66b68a0e4e3b5982dac1b3b279c319750b164413d9287abd0118cc421a74dcd5
+    manifestHash: 807d571aa1bbad0be466b206b4b226d86c6f8aa933ab344e1d03f22cd9b78df8
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -328,6 +328,8 @@ spec:
         - --skip-nodes-with-local-storage=true
         - --skip-nodes-with-system-pods=true
         - --scale-down-delay-after-add=10m0s
+        - --scale-down-unneeded-time=10m0s
+        - --scale-down-unready-time=20m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
         - --cordon-node-before-terminating=true

tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_cluster-completed.spec_content

@@ -38,6 +38,8 @@ spec:
     maxNodeProvisionTime: 15m0s
     newPodScaleUpDelay: 0s
     scaleDownDelayAfterAdd: 10m0s
+    scaleDownUnneededTime: 10m0s
+    scaleDownUnreadyTime: 20m0s
     scaleDownUtilizationThreshold: "0.5"
     skipNodesWithLocalStorage: true
     skipNodesWithSystemPods: true

tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 6afa3736e0b25db06f8644fa2c2dbad6704ced7056d48aa9d53e7e16e48de4bf
+    manifestHash: 9ac471700534766b341017aa1ac792a1df7e7cb5f60dabebd1395640d78621e8
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -328,6 +328,8 @@ spec:
         - --skip-nodes-with-local-storage=true
         - --skip-nodes-with-system-pods=true
         - --scale-down-delay-after-add=10m0s
+        - --scale-down-unneeded-time=10m0s
+        - --scale-down-unready-time=20m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
         - --cordon-node-before-terminating=true

tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content

@@ -37,6 +37,8 @@ spec:
     maxNodeProvisionTime: 15m0s
     newPodScaleUpDelay: 0s
     scaleDownDelayAfterAdd: 10m0s
+    scaleDownUnneededTime: 10m0s
+    scaleDownUnreadyTime: 20m0s
     scaleDownUtilizationThreshold: "0.5"
     skipNodesWithLocalStorage: true
     skipNodesWithSystemPods: true

tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 66644bc20f035c6c73f870100b8f083f266e851a80f76d8cc74240a7d852db4b
+    manifestHash: f230b1756485b2615617054687dd4e9d9e8a489afeedcac2b939dab8e8fe5b31
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -328,6 +328,8 @@ spec:
         - --skip-nodes-with-local-storage=true
         - --skip-nodes-with-system-pods=true
         - --scale-down-delay-after-add=10m0s
+        - --scale-down-unneeded-time=10m0s
+        - --scale-down-unready-time=20m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
         - --cordon-node-before-terminating=true

tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content

@@ -37,6 +37,8 @@ spec:
     maxNodeProvisionTime: 15m0s
     newPodScaleUpDelay: 0s
     scaleDownDelayAfterAdd: 10m0s
+    scaleDownUnneededTime: 10m0s
+    scaleDownUnreadyTime: 20m0s
     scaleDownUtilizationThreshold: "0.5"
     skipNodesWithLocalStorage: true
     skipNodesWithSystemPods: true

tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 66644bc20f035c6c73f870100b8f083f266e851a80f76d8cc74240a7d852db4b
+    manifestHash: f230b1756485b2615617054687dd4e9d9e8a489afeedcac2b939dab8e8fe5b31
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -328,6 +328,8 @@ spec:
         - --skip-nodes-with-local-storage=true
         - --skip-nodes-with-system-pods=true
         - --scale-down-delay-after-add=10m0s
+        - --scale-down-unneeded-time=10m0s
+        - --scale-down-unready-time=20m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
         - --cordon-node-before-terminating=true

tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content

@@ -37,6 +37,8 @@ spec:
     maxNodeProvisionTime: 15m0s
     newPodScaleUpDelay: 0s
     scaleDownDelayAfterAdd: 10m0s
+    scaleDownUnneededTime: 10m0s
+    scaleDownUnreadyTime: 20m0s
     scaleDownUtilizationThreshold: "0.5"
     skipNodesWithLocalStorage: true
     skipNodesWithSystemPods: true

tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 91ec876d9fef5d9dfd5ca7c9aca880da90d798c3894fb1699a610b7ff231b527
+    manifestHash: 552dda0544378cb79b23b1f1cf567ba8a4e0e27f8e57b88bf01e9cebfd23bbca
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -331,6 +331,8 @@ spec:
         - --skip-nodes-with-local-storage=true
         - --skip-nodes-with-system-pods=true
         - --scale-down-delay-after-add=10m0s
+        - --scale-down-unneeded-time=10m0s
+        - --scale-down-unready-time=20m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
         - --cordon-node-before-terminating=true

tests/integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content

@@ -30,6 +30,8 @@ spec:
     podAnnotations:
       testAnnotation: testAnnotation
     scaleDownDelayAfterAdd: 10m0s
+    scaleDownUnneededTime: 10m0s
+    scaleDownUnreadyTime: 20m0s
     scaleDownUtilizationThreshold: "0.5"
     skipNodesWithLocalStorage: true
     skipNodesWithSystemPods: true

tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 8908b36fc06477773fea4bf325ddd03215749ecb7906a62f5b911c449c0910cb
+    manifestHash: 10a9666c7d5a66522e82610f9e076abf531a758315dd1579390540559219b30f
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -332,6 +332,8 @@ spec:
         - --skip-nodes-with-local-storage=true
         - --skip-nodes-with-system-pods=true
         - --scale-down-delay-after-add=10m0s
+        - --scale-down-unneeded-time=10m0s
+        - --scale-down-unready-time=20m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
         - --cordon-node-before-terminating=true

upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template

@@ -327,6 +327,8 @@ spec:
             - --skip-nodes-with-local-storage={{ .SkipNodesWithLocalStorage }}
             - --skip-nodes-with-system-pods={{ .SkipNodesWithSystemPods }}
             - --scale-down-delay-after-add={{ .ScaleDownDelayAfterAdd }}
+            - --scale-down-unneeded-time={{ .ScaleDownUnneededTime }}
+            - --scale-down-unready-time={{ .ScaleDownUnreadyTime }}
             - --new-pod-scale-up-delay={{ .NewPodScaleUpDelay }}
             - --max-node-provision-time={{ .MaxNodeProvisionTime }}
             # This flag does not exist before CAS 1.21