Merge pull request #16693 from alexandresavicki/master

feat(cluster-autoscaler) Implement emitPerNodegroupMetrics parameter

docs/addons.md

@@ -71,6 +71,7 @@ spec:
     enabled: true
     expander: least-waste
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     awsUseStaticInstanceList: false
     scaleDownUtilizationThreshold: 0.5
     skipNodesWithCustomControllerPods: true

docs/releases/1.29-NOTES.md

@@ -2,6 +2,8 @@
 
 # Significant changes
 
+* Add support to --emit-per-nodegroup-metrics on the cluster autoscaler addon (emitPerNodegroupMetrics) [PR#16693](https://github.com/kubernetes/kops/pull/16693/)
+
 ## Deferred deletion / pruning phase
 
 Some infrastructure changes are potentially disruptive to the continued

k8s/crds/kops.k8s.io_clusters.yaml

@@ -769,6 +769,11 @@ spec:
                       CustomPriorityExpanderConfig overides the priority-expander ConfigMap with the provided configuration. Any InstanceGroup configuration will be ignored if this is set.
                       This could be useful in order to use regex on priorities configuration
                     type: object
+                  emitPerNodegroupMetrics:
+                    description: |-
+                      EmitPerNodegroupMetrics If true, publishes the node groups min and max metrics count set on the cluster autoscaler.
+                      Default: false
+                    type: boolean
                   enabled:
                     description: |-
                       Enabled enables the cluster autoscaler.

pkg/apis/kops/componentconfig.go

@@ -1088,6 +1088,9 @@ type ClusterAutoscalerConfig struct {
 	// BalanceSimilarNodeGroups makes the cluster autoscaler treat similar node groups as one.
 	// Default: false
 	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
+	// EmitPerNodegroupMetrics If true, publishes the node groups min and max metrics count set on the cluster autoscaler.
+	// Default: false
+	EmitPerNodegroupMetrics *bool `json:"emitPerNodegroupMetrics,omitempty"`
 	// AWSUseStaticInstanceList makes cluster autoscaler to use statically defined set of AWS EC2 Instance List.
 	// Default: false
 	AWSUseStaticInstanceList *bool `json:"awsUseStaticInstanceList,omitempty"`

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -1151,6 +1151,9 @@ type ClusterAutoscalerConfig struct {
 	// BalanceSimilarNodeGroups makes the cluster autoscaler treat similar node groups as one.
 	// Default: false
 	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
+	// EmitPerNodegroupMetrics If true, publishes the node groups min and max metrics count set on the cluster autoscaler.
+	// Default: false
+	EmitPerNodegroupMetrics *bool `json:"emitPerNodegroupMetrics,omitempty"`
 	// AWSUseStaticInstanceList makes the cluster autoscaler to use statically defined set of AWS EC2 Instance List.
 	// Default: false
 	AWSUseStaticInstanceList *bool `json:"awsUseStaticInstanceList,omitempty"`

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -2313,6 +2313,7 @@ func autoConvert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfi
 	out.Enabled = in.Enabled
 	out.Expander = in.Expander
 	out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups
+	out.EmitPerNodegroupMetrics = in.EmitPerNodegroupMetrics
 	out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList
 	out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization
 	out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold
@@ -2343,6 +2344,7 @@ func autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfi
 	out.Enabled = in.Enabled
 	out.Expander = in.Expander
 	out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups
+	out.EmitPerNodegroupMetrics = in.EmitPerNodegroupMetrics
 	out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList
 	out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization
 	out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -935,6 +935,11 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EmitPerNodegroupMetrics != nil {
+		in, out := &in.EmitPerNodegroupMetrics, &out.EmitPerNodegroupMetrics
+		*out = new(bool)
+		**out = **in
+	}
 	if in.AWSUseStaticInstanceList != nil {
 		in, out := &in.AWSUseStaticInstanceList, &out.AWSUseStaticInstanceList
 		*out = new(bool)

pkg/apis/kops/v1alpha3/componentconfig.go

@@ -1080,6 +1080,9 @@ type ClusterAutoscalerConfig struct {
 	// BalanceSimilarNodeGroups makes the cluster autoscaler treat similar node groups as one.
 	// Default: false
 	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
+	// EmitPerNodegroupMetrics If true, publishes the node groups min and max metrics count set on the cluster autoscaler.
+	// Default: false
+	EmitPerNodegroupMetrics *bool `json:"emitPerNodegroupMetrics,omitempty"`
 	// AWSUseStaticInstanceList makes the cluster autoscaler to use statically defined set of AWS EC2 Instance List.
 	// Default: false
 	AWSUseStaticInstanceList *bool `json:"awsUseStaticInstanceList,omitempty"`

pkg/apis/kops/v1alpha3/zz_generated.conversion.go

@@ -2559,6 +2559,7 @@ func autoConvert_v1alpha3_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfi
 	out.Enabled = in.Enabled
 	out.Expander = in.Expander
 	out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups
+	out.EmitPerNodegroupMetrics = in.EmitPerNodegroupMetrics
 	out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList
 	out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization
 	out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold
@@ -2589,6 +2590,7 @@ func autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha3_ClusterAutoscalerConfi
 	out.Enabled = in.Enabled
 	out.Expander = in.Expander
 	out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups
+	out.EmitPerNodegroupMetrics = in.EmitPerNodegroupMetrics
 	out.AWSUseStaticInstanceList = in.AWSUseStaticInstanceList
 	out.IgnoreDaemonSetsUtilization = in.IgnoreDaemonSetsUtilization
 	out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold

pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go

@@ -897,6 +897,11 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EmitPerNodegroupMetrics != nil {
+		in, out := &in.EmitPerNodegroupMetrics, &out.EmitPerNodegroupMetrics
+		*out = new(bool)
+		**out = **in
+	}
 	if in.AWSUseStaticInstanceList != nil {
 		in, out := &in.AWSUseStaticInstanceList, &out.AWSUseStaticInstanceList
 		*out = new(bool)

pkg/apis/kops/zz_generated.deepcopy.go

@@ -994,6 +994,11 @@ func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EmitPerNodegroupMetrics != nil {
+		in, out := &in.EmitPerNodegroupMetrics, &out.EmitPerNodegroupMetrics
+		*out = new(bool)
+		**out = **in
+	}
 	if in.AWSUseStaticInstanceList != nil {
 		in, out := &in.AWSUseStaticInstanceList, &out.AWSUseStaticInstanceList
 		*out = new(bool)

pkg/model/components/clusterautoscaler.go

@@ -83,6 +83,9 @@ func (b *ClusterAutoscalerOptionsBuilder) BuildOptions(o interface{}) error {
 	if cas.BalanceSimilarNodeGroups == nil {
 		cas.BalanceSimilarNodeGroups = fi.PtrTo(false)
 	}
+	if cas.EmitPerNodegroupMetrics == nil {
+		cas.EmitPerNodegroupMetrics = fi.PtrTo(false)
+	}
 	if cas.AWSUseStaticInstanceList == nil {
 		cas.AWSUseStaticInstanceList = fi.PtrTo(false)
 	}

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 963767d82e85200787e166f3cc6e7b3b9e60d2383c9fef71a562d9e4eedd7086
+    manifestHash: e4817b8481df3f556c19999b552e60c6305ce8063481c6ec5d987f225aa686ad
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -350,6 +350,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=priority

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content

@@ -34,6 +34,7 @@ spec:
       - .*low.*
       "100":
       - .*high.*
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: priority
     ignoreDaemonSetsUtilization: false

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: d28605db2e6ade6fee371d462c0e9893871ea73a35ad598fa616a2657290d319
+    manifestHash: 8c697d6e92263502eeeb637b47669c5fbb3ba11cd82c90a653cddb42060e7fe5
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -350,6 +350,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=priority

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content

@@ -27,6 +27,7 @@ spec:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
     createPriorityExpanderConfig: true
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: priority
     ignoreDaemonSetsUtilization: false

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content

@@ -30,6 +30,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 10a6f07c81b4244162cf161255fd1aae420c1eada3e262aabe4f6930b2f5a65f
+    manifestHash: 3764bb2698fcd6ad15853cf437c1664e7f46acfee77fdc3c686f1aa000ab4006
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -321,6 +321,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content

@@ -30,6 +30,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 114e13595bf4571c767f786144e013bc2b100fa35d23c83ecc8af940c4455cf4
+    manifestHash: c03c6c1e8806b7e572506d2018bdbb9c314f5bc207473d85e95e7308af4d5193
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -321,6 +321,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content

@@ -30,6 +30,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -49,7 +49,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 10a6f07c81b4244162cf161255fd1aae420c1eada3e262aabe4f6930b2f5a65f
+    manifestHash: 3764bb2698fcd6ad15853cf437c1664e7f46acfee77fdc3c686f1aa000ab4006
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -321,6 +321,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content

@@ -31,6 +31,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 39af22339588beb7c2fd4b2b5c9e060a6c84a9fc2d8efac9b078a9ffff64753f
+    manifestHash: 45d2bf5133e21bd182463572db9f524648214e3123c8273609d0112cd9ca925a
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -327,6 +327,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content

@@ -31,6 +31,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 83138df304119f6137709766b9f76c23743f5cfc2a0d9400fb6e51444e15fde4
+    manifestHash: a60534cf6f674d86d17a32490fef182ebf5d821523402ea8cbb1d589eaec5031
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -327,6 +327,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=gce
         - --expander=random
         - --nodes=1:1:https://www.googleapis.com/compute/v1/projects/testproject/zones/us-test1-a/instanceGroups/a-nodes-minimal-example-com

tests/integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content

@@ -31,6 +31,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

tests/integration/update_cluster/many-addons/data/aws_s3_object_many-addons.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 48d9a078fde2ad3617492f8bae23edac27fdf0d9b6102d298623b8ccf010a708
+    manifestHash: 44be0298541ea2f643250d74df219ea0846304d14bbb49529773830e028106e9
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

tests/integration/update_cluster/many-addons/data/aws_s3_object_many-addons.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content

@@ -328,6 +328,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template

@@ -333,6 +333,7 @@ spec:
           command:
             - ./cluster-autoscaler
             - --balance-similar-node-groups={{ .BalanceSimilarNodeGroups }}
+            - --emit-per-nodegroup-metrics={{ .EmitPerNodegroupMetrics }}
             - --cloud-provider={{ GetCloudProvider }}
             {{ if (eq GetCloudProvider "aws") }}
             - --aws-use-static-instance-list={{ .AWSUseStaticInstanceList }}