Allow Cluster Autoscaler to ignore daemon pods

By default the cluster autoscaler takes DaemonSet-managed pods' resource requests into consideration when computing a node's resource utilization. Allow toggling its "--ignore-daemonsets-utilization" command-line flag via a new field in the Cluster spec—"clusterAutoscaler.ignoreDaemonSetsUtilization." Setting that field to true causes the autoscaler to ignore such daemon pods' requests, such that it will more likely judge a node running only daemon pods as being underutilized and shut down its hosting machine.
2023-04-04 11:00:59 -04:00 · 2023-04-04 11:00:59 -04:00 · 9595c833ee
parent c8c2abfc97
commit 9595c833ee
41 changed files with 90 additions and 33 deletions
--- a/k8s/crds/kops.k8s.io_clusters.yaml
+++ b/k8s/crds/kops.k8s.io_clusters.yaml
@ -680,12 +680,12 @@ spec:
                description: ClusterAutoscaler defines the cluaster autoscaler configuration.
                properties:
                  awsUseStaticInstanceList:
-                    description: 'AWSUseStaticInstanceList makes cluster autoscaler
+                    description: 'AWSUseStaticInstanceList makes the cluster autoscaler
                      to use statically defined set of AWS EC2 Instance List. Default:
                      false'
                    type: boolean
                  balanceSimilarNodeGroups:
-                    description: 'BalanceSimilarNodeGroups makes cluster autoscaler
+                    description: 'BalanceSimilarNodeGroups makes the cluster autoscaler
                      treat similar node groups as one. Default: false'
                    type: boolean
                  cordonNodeBeforeTerminating:
@ -726,6 +726,11 @@ spec:
                      ConfigMap based on the `autoscale` and `autoscalePriority` fields
                      in the InstanceGroup specs. Default: least-waste'
                    type: string
+                  ignoreDaemonSetsUtilization:
+                    description: 'IgnoreDaemonSetsUtilization causes the cluster autoscaler
+                      to ignore DaemonSet-managed pods when calculating resource utilization
+                      for scaling down. Default: false'
+                    type: boolean
                  image:
                    description: 'Image is the docker container used. Default: the
                      latest supported image for the specified kubernetes version.'
@ -743,15 +748,15 @@ spec:
                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                    x-kubernetes-int-or-string: true
                  newPodScaleUpDelay:
-                    description: 'NewPodScaleUpDelay causes cluster autoscaler to
-                      ignore unschedulable pods until they are a certain "age", regardless
-                      of the scan-interval Default: 0s'
+                    description: 'NewPodScaleUpDelay causes the cluster autoscaler
+                      to ignore unschedulable pods until they are a certain "age",
+                      regardless of the scan-interval Default: 0s'
                    type: string
                  podAnnotations:
                    additionalProperties:
                      type: string
                    description: 'PodAnnotations are the annotations added to cluster
-                      autoscaler pod when they are created. Default: none'
+                      autoscaler pods when they are created. Default: none'
                    type: object
                  scaleDownDelayAfterAdd:
                    description: 'ScaleDownDelayAfterAdd determines the time after
@ -772,11 +777,11 @@ spec:
                      threshold for node scale-down. Default: 0.5'
                    type: string
                  skipNodesWithLocalStorage:
-                    description: 'SkipNodesWithLocalStorage makes cluster autoscaler
+                    description: 'SkipNodesWithLocalStorage makes the cluster autoscaler
                      skip scale-down of nodes with local storage. Default: true'
                    type: boolean
                  skipNodesWithSystemPods:
-                    description: 'SkipNodesWithSystemPods makes cluster autoscaler
+                    description: 'SkipNodesWithSystemPods makes the cluster autoscaler
                      skip scale-down of nodes with non-DaemonSet pods in the kube-system
                      namespace. Default: true'
                    type: boolean
--- a/pkg/apis/kops/componentconfig.go
+++ b/pkg/apis/kops/componentconfig.go
@ -990,22 +990,25 @@ type ClusterAutoscalerConfig struct {
 	// By default, kOps will generate the priority expander ConfigMap based on the `autoscale` and `autoscalePriority` fields in the InstanceGroup specs.
 	// Default: least-waste
 	Expander string `json:"expander,omitempty"`
-	// BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one.
+	// BalanceSimilarNodeGroups makes the cluster autoscaler treat similar node groups as one.
 	// Default: false
 	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
 	// AWSUseStaticInstanceList makes cluster autoscaler to use statically defined set of AWS EC2 Instance List.
 	// Default: false
 	AWSUseStaticInstanceList *bool `json:"awsUseStaticInstanceList,omitempty"`
+	// IgnoreDaemonSetsUtilization causes the cluster autoscaler to ignore DaemonSet-managed pods when calculating resource utilization for scaling down.
+	// Default: false
+	IgnoreDaemonSetsUtilization *bool `json:"ignoreDaemonSetsUtilization,omitempty"`
 	// ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down.
 	// Default: 0.5
 	ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"`
-	// SkipNodesWithSystemPods makes cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace.
+	// SkipNodesWithSystemPods makes the cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace.
 	// Default: true
 	SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"`
-	// SkipNodesWithLocalStorage makes cluster autoscaler skip scale-down of nodes with local storage.
+	// SkipNodesWithLocalStorage makes the cluster autoscaler skip scale-down of nodes with local storage.
 	// Default: true
 	SkipNodesWithLocalStorage *bool `json:"skipNodesWithLocalStorage,omitempty"`
-	// NewPodScaleUpDelay causes cluster autoscaler to ignore unschedulable pods until they are a certain "age", regardless of the scan-interval
+	// NewPodScaleUpDelay causes the cluster autoscaler to ignore unschedulable pods until they are a certain "age", regardless of the scan-interval
 	// Default: 0s
 	NewPodScaleUpDelay *string `json:"newPodScaleUpDelay,omitempty"`
 	// ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes
--- a/pkg/apis/kops/v1alpha2/componentconfig.go
+++ b/pkg/apis/kops/v1alpha2/componentconfig.go
@ -1053,22 +1053,25 @@ type ClusterAutoscalerConfig struct {
 	// By default, kOps will generate the priority expander ConfigMap based on the `autoscale` and `autoscalePriority` fields in the InstanceGroup specs.
 	// Default: least-waste
 	Expander string `json:"expander,omitempty"`
-	// BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one.
+	// BalanceSimilarNodeGroups makes the cluster autoscaler treat similar node groups as one.
 	// Default: false
 	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
-	// AWSUseStaticInstanceList makes cluster autoscaler to use statically defined set of AWS EC2 Instance List.
+	// AWSUseStaticInstanceList makes the cluster autoscaler to use statically defined set of AWS EC2 Instance List.
 	// Default: false
 	AWSUseStaticInstanceList *bool `json:"awsUseStaticInstanceList,omitempty"`
+	// IgnoreDaemonSetsUtilization causes the cluster autoscaler to ignore DaemonSet-managed pods when calculating resource utilization for scaling down.
+	// Default: false
+	IgnoreDaemonSetsUtilization *bool `json:"ignoreDaemonSetsUtilization,omitempty"`
 	// ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down.
 	// Default: 0.5
 	ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"`
-	// SkipNodesWithSystemPods makes cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace.
+	// SkipNodesWithSystemPods makes the cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace.
 	// Default: true
 	SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"`
-	// SkipNodesWithLocalStorage makes cluster autoscaler skip scale-down of nodes with local storage.
+	// SkipNodesWithLocalStorage makes the cluster autoscaler skip scale-down of nodes with local storage.
 	// Default: true
 	SkipNodesWithLocalStorage *bool `json:"skipNodesWithLocalStorage,omitempty"`
-	// NewPodScaleUpDelay causes cluster autoscaler to ignore unschedulable pods until they are a certain "age", regardless of the scan-interval
+	// NewPodScaleUpDelay causes the cluster autoscaler to ignore unschedulable pods until they are a certain "age", regardless of the scan-interval
 	// Default: 0s
 	NewPodScaleUpDelay *string `json:"newPodScaleUpDelay,omitempty"`
 	// ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes
@ -1094,7 +1097,7 @@ type ClusterAutoscalerConfig struct {
 	CPURequest *resource.Quantity `json:"cpuRequest,omitempty"`
 	// MaxNodeProvisionTime determines how long CAS will wait for a node to join the cluster.
 	MaxNodeProvisionTime string `json:"maxNodeProvisionTime,omitempty"`
-	// PodAnnotations are the annotations added to cluster autoscaler pod when they are created.
+	// PodAnnotations are the annotations added to cluster autoscaler pods when they are created.
 	// Default: none
 	PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
 	// CreatePriorityExpenderConfig makes kOps create the priority-expander ConfigMap
--- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
@ -2230,6 +2230,7 @@ func autoConvert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfi
 	out.Expander = in.Expander
 	out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups
 	out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList
+	out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization
 	out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold
 	out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods
 	out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage
@ -2258,6 +2259,7 @@ func autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfi
 	out.Expander = in.Expander
 	out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups
 	out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList
+	out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization
 	out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold
 	out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods
 	out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage
--- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
@ -885,6 +885,11 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.IgnoreDaemonSetsUtilization != nil {
+		in, out := &in.IgnoreDaemonSetsUtilization, &out.IgnoreDaemonSetsUtilization
+		*out = new(bool)
+		**out = **in
+	}
 	if in.ScaleDownUtilizationThreshold != nil {
 		in, out := &in.ScaleDownUtilizationThreshold, &out.ScaleDownUtilizationThreshold
 		*out = new(string)
--- a/pkg/apis/kops/v1alpha3/componentconfig.go
+++ b/pkg/apis/kops/v1alpha3/componentconfig.go
@ -983,22 +983,25 @@ type ClusterAutoscalerConfig struct {
 	// By default, kOps will generate the priority expander ConfigMap based on the `autoscale` and `autoscalePriority` fields in the InstanceGroup specs.
 	// Default: least-waste
 	Expander string `json:"expander,omitempty"`
-	// BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one.
+	// BalanceSimilarNodeGroups makes the cluster autoscaler treat similar node groups as one.
 	// Default: false
 	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
-	// AWSUseStaticInstanceList makes cluster autoscaler to use statically defined set of AWS EC2 Instance List.
+	// AWSUseStaticInstanceList makes the cluster autoscaler to use statically defined set of AWS EC2 Instance List.
 	// Default: false
 	AWSUseStaticInstanceList *bool `json:"awsUseStaticInstanceList,omitempty"`
+	// IgnoreDaemonSetsUtilization causes the cluster autoscaler to ignore DaemonSet-managed pods when calculating resource utilization for scaling down.
+	// Default: false
+	IgnoreDaemonSetsUtilization *bool `json:"ignoreDaemonSetsUtilization,omitempty"`
 	// ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down.
 	// Default: 0.5
 	ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"`
-	// SkipNodesWithSystemPods makes cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace.
+	// SkipNodesWithSystemPods makes the cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace.
 	// Default: true
 	SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"`
-	// SkipNodesWithLocalStorage makes cluster autoscaler skip scale-down of nodes with local storage.
+	// SkipNodesWithLocalStorage makes the cluster autoscaler skip scale-down of nodes with local storage.
 	// Default: true
 	SkipNodesWithLocalStorage *bool `json:"skipNodesWithLocalStorage,omitempty"`
-	// NewPodScaleUpDelay causes cluster autoscaler to ignore unschedulable pods until they are a certain "age", regardless of the scan-interval
+	// NewPodScaleUpDelay causes the cluster autoscaler to ignore unschedulable pods until they are a certain "age", regardless of the scan-interval
 	// Default: 0s
 	NewPodScaleUpDelay *string `json:"newPodScaleUpDelay,omitempty"`
 	// ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes
--- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
@ -2464,6 +2464,7 @@ func autoConvert_v1alpha3_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfi
 	out.Expander = in.Expander
 	out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups
 	out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList
+	out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization
 	out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold
 	out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods
 	out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage
@ -2492,6 +2493,7 @@ func autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha3_ClusterAutoscalerConfi
 	out.Expander = in.Expander
 	out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups
 	out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList
+	out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization
 	out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold
 	out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods
 	out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage
--- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
@ -842,6 +842,11 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.IgnoreDaemonSetsUtilization != nil {
+		in, out := &in.IgnoreDaemonSetsUtilization, &out.IgnoreDaemonSetsUtilization
+		*out = new(bool)
+		**out = **in
+	}
 	if in.ScaleDownUtilizationThreshold != nil {
 		in, out := &in.ScaleDownUtilizationThreshold, &out.ScaleDownUtilizationThreshold
 		*out = new(string)
--- a/pkg/apis/kops/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/zz_generated.deepcopy.go
@ -939,6 +939,11 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.IgnoreDaemonSetsUtilization != nil {
+		in, out := &in.IgnoreDaemonSetsUtilization, &out.IgnoreDaemonSetsUtilization
+		*out = new(bool)
+		**out = **in
+	}
 	if in.ScaleDownUtilizationThreshold != nil {
 		in, out := &in.ScaleDownUtilizationThreshold, &out.ScaleDownUtilizationThreshold
 		*out = new(string)
--- a/pkg/model/components/clusterautoscaler.go
+++ b/pkg/model/components/clusterautoscaler.go
@ -63,6 +63,9 @@ func (b *ClusterAutoscalerOptionsBuilder) BuildOptions(o interface{}) error {
 	if cas.Expander == "" {
 		cas.Expander = "random"
 	}
+	if cas.IgnoreDaemonSetsUtilization == nil {
+		cas.IgnoreDaemonSetsUtilization = fi.PtrTo(false)
+	}
 	if cas.ScaleDownUtilizationThreshold == nil {
 		cas.ScaleDownUtilizationThreshold = fi.PtrTo("0.5")
 	}
--- a/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-bootstrap_content
@ -41,7 +41,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: e485760f38e39e371e1253b9ff924501e22438031e62f2c1188b0b1c7bd41d0a
+    manifestHash: d0155a29604d8ceb39798066ad7ad965f540488968df34a361bea4fdf2ee5388
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -356,6 +356,7 @@ spec:
        - --nodes=2:2:nodes.cas-priority-expander-custom.example.com
        - --nodes=2:2:nodes-high-priority.cas-priority-expander-custom.example.com
        - --nodes=2:2:nodes-low-priority.cas-priority-expander-custom.example.com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content
@ -37,6 +37,7 @@ spec:
      - .*high.*
    enabled: true
    expander: priority
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.1
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-bootstrap_content
@ -41,7 +41,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: bb98d9d34142130dbaf95afcd9001340e7437a21f6c0915dedff4b62132879ee
+    manifestHash: f20652532bf2fe33a36ed1ba75d04d10c87286c1b9f86c38a975fc6005c8b1e8
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -356,6 +356,7 @@ spec:
        - --nodes=2:2:nodes.cas-priority-expander.example.com
        - --nodes=2:2:nodes-high-priority.cas-priority-expander.example.com
        - --nodes=2:2:nodes-low-priority.cas-priority-expander.example.com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content
@ -30,6 +30,7 @@ spec:
    createPriorityExpanderConfig: true
    enabled: true
    expander: priority
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.1
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content
@ -33,6 +33,7 @@ spec:
    balanceSimilarNodeGroups: false
    enabled: true
    expander: random
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.1
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@ -41,7 +41,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: ae5578d4a54d4c7fd6d4ffc55c0b382c6b3dce39f258dd5ddc17d726d3b797db
+    manifestHash: f2143d6228cb74163f545f0269f2e54b4562193b3624634a1920d34d7053a97b
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -325,6 +325,7 @@ spec:
        - --aws-use-static-instance-list=false
        - --expander=random
        - --nodes=2:2:nodes.minimal.example.com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_cluster-completed.spec_content
@ -33,6 +33,7 @@ spec:
    balanceSimilarNodeGroups: false
    enabled: true
    expander: random
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.23.1
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@ -48,7 +48,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: d8e860b6e887d2e05b326668f6961d6b05f6d05808c2427364adc593ae699087
+    manifestHash: 95e8f3f3df4c8b54c4c40ee52626ef2816e87be067cb7938b6eaad25d08bca11
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -329,6 +329,7 @@ spec:
        - --aws-use-static-instance-list=false
        - --expander=random
        - --nodes=2:2:nodes.minimal.example.com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_cluster-completed.spec_content
@ -34,6 +34,7 @@ spec:
    balanceSimilarNodeGroups: false
    enabled: true
    expander: random
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.24.0
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@ -48,7 +48,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: dd347939c0818fa95b0c56f78135668466b66e14d0d73c58cde3d14a7c13feec
+    manifestHash: e4bff9a2ea75d0998cbaf50cb4ec75a70a4b9e455fcee7ffa5acefabea6560a2
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -325,6 +325,7 @@ spec:
        - --aws-use-static-instance-list=false
        - --expander=random
        - --nodes=2:2:nodes.minimal.example.com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content
@ -33,6 +33,7 @@ spec:
    balanceSimilarNodeGroups: false
    enabled: true
    expander: random
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.25.0
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@ -48,7 +48,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 7b18a30ca3e842db84aabd59434c8d2c7de35a38c9ebaff9ba203b4cc3a82e1b
+    manifestHash: 938a4d98c22dea2da0244e52cf99986b09b097115baa604b43d54346f632ac80
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -325,6 +325,7 @@ spec:
        - --aws-use-static-instance-list=false
        - --expander=random
        - --nodes=2:2:nodes.minimal.example.com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content
@ -33,6 +33,7 @@ spec:
    balanceSimilarNodeGroups: false
    enabled: true
    expander: random
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.1
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@ -49,7 +49,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: ae5578d4a54d4c7fd6d4ffc55c0b382c6b3dce39f258dd5ddc17d726d3b797db
+    manifestHash: f2143d6228cb74163f545f0269f2e54b4562193b3624634a1920d34d7053a97b
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -325,6 +325,7 @@ spec:
        - --aws-use-static-instance-list=false
        - --expander=random
        - --nodes=2:2:nodes.minimal.example.com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content
@ -33,6 +33,7 @@ spec:
    balanceSimilarNodeGroups: false
    enabled: true
    expander: random
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.1
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@ -41,7 +41,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 0030f0d28daca7e802e7c77d817b67a007afbf5781850feb31c8c06bfa8c1719
+    manifestHash: a42f68cd53cd69bccc7e1e19e99569b038d3d5d360e3cef21161c2f21b9bd704
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -331,6 +331,7 @@ spec:
        - --aws-use-static-instance-list=false
        - --expander=random
        - --nodes=2:2:nodes.minimal.example.com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content
@ -33,6 +33,7 @@ spec:
    balanceSimilarNodeGroups: false
    enabled: true
    expander: random
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.1
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@ -41,7 +41,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: d97c26afc32f2dedaa46687d789350960158fcfc7e2d016ef79ad20ea8bfa7c2
+    manifestHash: ff5b90b8081567456315dacd3a1796048f3277529ca9f1a6e31ea80a5c47ed30
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -330,6 +330,7 @@ spec:
        - --cloud-provider=gce
        - --expander=random
        - --nodes=1:1:https://www.googleapis.com/compute/v1/projects/testproject/zones/us-test1-a/instanceGroups/a-nodes-minimal-example-com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/tests/integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content
@ -32,6 +32,7 @@ spec:
    balanceSimilarNodeGroups: false
    enabled: true
    expander: random
+    ignoreDaemonSetsUtilization: false
    image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.1
    maxNodeProvisionTime: 15m0s
    newPodScaleUpDelay: 0s
--- a/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@ -41,7 +41,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.15
    manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 00428a36366e0731a034a128cb72769cad294ab9fcc430d93261b6ebadcb0651
+    manifestHash: 4d573f9ce40fdb889103a77b1ae27e0adeda65e6a820dc876e968657a4df2592
    name: cluster-autoscaler.addons.k8s.io
    selector:
      k8s-addon: cluster-autoscaler.addons.k8s.io
--- a/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
+++ b/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@ -332,6 +332,7 @@ spec:
        - --aws-use-static-instance-list=false
        - --expander=random
        - --nodes=2:2:nodes.minimal.example.com
+        - --ignore-daemonsets-utilization=false
        - --scale-down-utilization-threshold=0.5
        - --skip-nodes-with-local-storage=true
        - --skip-nodes-with-system-pods=true
--- a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template
+++ b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template
@ -346,6 +346,7 @@ spec:
            {{ range $nodeGroup := GetClusterAutoscalerNodeGroups }}
            - --nodes={{ $nodeGroup.MinSize }}:{{ $nodeGroup.MaxSize }}:{{ $nodeGroup.Other }}
            {{ end }}
+            - --ignore-daemonsets-utilization={{ .IgnoreDaemonSetsUtilization }}
            - --scale-down-utilization-threshold={{ .ScaleDownUtilizationThreshold }}
            - --skip-nodes-with-local-storage={{ .SkipNodesWithLocalStorage }}
            - --skip-nodes-with-system-pods={{ .SkipNodesWithSystemPods }}