diff --git a/pkg/cmd/autoscale/autoscale.go b/pkg/cmd/autoscale/autoscale.go index 51fbf3ba..8a197016 100644 --- a/pkg/cmd/autoscale/autoscale.go +++ b/pkg/cmd/autoscale/autoscale.go @@ -24,13 +24,17 @@ import ( "k8s.io/klog/v2" autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/genericiooptions" "k8s.io/cli-runtime/pkg/printers" "k8s.io/cli-runtime/pkg/resource" autoscalingv1client "k8s.io/client-go/kubernetes/typed/autoscaling/v1" + autoscalingv2client "k8s.io/client-go/kubernetes/typed/autoscaling/v2" "k8s.io/client-go/scale" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/scheme" @@ -43,6 +47,7 @@ import ( var ( autoscaleLong = templates.LongDesc(i18n.T(` Creates an autoscaler that automatically chooses and sets the number of pods that run in a Kubernetes cluster. + The command will attempt to use the autoscaling/v2 API first, in case of an error, it will fall back to autoscaling/v1 API. Looks up a deployment, replica set, stateful set, or replication controller by name and creates an autoscaler that uses the given resource as a reference. An autoscaler can automatically increase or decrease number of pods deployed within the system as needed.`)) @@ -78,7 +83,8 @@ type AutoscaleOptions struct { builder *resource.Builder fieldManager string - HPAClient autoscalingv1client.HorizontalPodAutoscalersGetter + HPAClientV1 autoscalingv1client.HorizontalPodAutoscalersGetter + HPAClientV2 autoscalingv2client.HorizontalPodAutoscalersGetter scaleKindResolver scale.ScaleKindResolver genericiooptions.IOStreams @@ -157,7 +163,8 @@ func (o *AutoscaleOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args if err != nil { return err } - o.HPAClient = kubeClient.AutoscalingV1() + o.HPAClientV2 = kubeClient.AutoscalingV2() + o.HPAClientV1 = kubeClient.AutoscalingV1() o.namespace, o.enforceNamespace, err = f.ToRawKubeConfigLoader().Namespace() if err != nil { @@ -186,7 +193,6 @@ func (o *AutoscaleOptions) Validate() error { return nil } -// Run performs the execution func (o *AutoscaleOptions) Run() error { r := o.builder. Unstructured(). @@ -212,44 +218,19 @@ func (o *AutoscaleOptions) Run() error { return fmt.Errorf("cannot autoscale a %v: %v", mapping.GroupVersionKind.Kind, err) } - hpa := o.createHorizontalPodAutoscaler(info.Name, mapping) - - if err := o.Recorder.Record(hpa); err != nil { - klog.V(4).Infof("error recording current command: %v", err) - } - - if o.dryRunStrategy == cmdutil.DryRunClient { - count++ - - printer, err := o.ToPrinter("created") - if err != nil { + // handles the creation of HorizontalPodAutoscaler objects for both v2 and v1 APIs. + // If v2 API fails, try to create and handle HorizontalPodAutoscaler using v1 API + hpaV2 := o.createHorizontalPodAutoscalerV2(info.Name, mapping) + if err := o.handleHPA(hpaV2); err != nil { + klog.V(1).Infof("Encountered an error with the v2 HorizontalPodAutoscaler: %v. "+ + "Falling back to try the v1 HorizontalPodAutoscaler", err) + hpaV1 := o.createHorizontalPodAutoscalerV1(info.Name, mapping) + if err := o.handleHPA(hpaV1); err != nil { return err } - return printer.PrintObj(hpa, o.Out) } - - if err := util.CreateOrUpdateAnnotation(o.createAnnotation, hpa, scheme.DefaultJSONEncoder()); err != nil { - return err - } - - createOptions := metav1.CreateOptions{} - if o.fieldManager != "" { - createOptions.FieldManager = o.fieldManager - } - if o.dryRunStrategy == cmdutil.DryRunServer { - createOptions.DryRun = []string{metav1.DryRunAll} - } - actualHPA, err := o.HPAClient.HorizontalPodAutoscalers(o.namespace).Create(context.TODO(), hpa, createOptions) - if err != nil { - return err - } - count++ - printer, err := o.ToPrinter("autoscaled") - if err != nil { - return err - } - return printer.PrintObj(actualHPA, o.Out) + return nil }) if err != nil { return err @@ -260,7 +241,96 @@ func (o *AutoscaleOptions) Run() error { return nil } -func (o *AutoscaleOptions) createHorizontalPodAutoscaler(refName string, mapping *meta.RESTMapping) *autoscalingv1.HorizontalPodAutoscaler { +// handleHPA handles the creation and management of a single HPA object. +func (o *AutoscaleOptions) handleHPA(hpa runtime.Object) error { + if err := o.Recorder.Record(hpa); err != nil { + return fmt.Errorf("error recording current command: %w", err) + } + + if o.dryRunStrategy == cmdutil.DryRunClient { + printer, err := o.ToPrinter("created") + if err != nil { + return err + } + return printer.PrintObj(hpa, o.Out) + } + + if err := util.CreateOrUpdateAnnotation(o.createAnnotation, hpa, scheme.DefaultJSONEncoder()); err != nil { + return err + } + + createOptions := metav1.CreateOptions{} + if o.fieldManager != "" { + createOptions.FieldManager = o.fieldManager + } + if o.dryRunStrategy == cmdutil.DryRunServer { + createOptions.DryRun = []string{metav1.DryRunAll} + } + + var actualHPA runtime.Object + var err error + switch typedHPA := hpa.(type) { + case *autoscalingv2.HorizontalPodAutoscaler: + actualHPA, err = o.HPAClientV2.HorizontalPodAutoscalers(o.namespace).Create(context.TODO(), typedHPA, createOptions) + case *autoscalingv1.HorizontalPodAutoscaler: + actualHPA, err = o.HPAClientV1.HorizontalPodAutoscalers(o.namespace).Create(context.TODO(), typedHPA, createOptions) + default: + return fmt.Errorf("unsupported HorizontalPodAutoscaler type %T", hpa) + } + if err != nil { + return err + } + + printer, err := o.ToPrinter("autoscaled") + if err != nil { + return err + } + return printer.PrintObj(actualHPA, o.Out) +} + +func (o *AutoscaleOptions) createHorizontalPodAutoscalerV2(refName string, mapping *meta.RESTMapping) *autoscalingv2.HorizontalPodAutoscaler { + name := o.Name + if len(name) == 0 { + name = refName + } + + scaler := autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: mapping.GroupVersionKind.GroupVersion().String(), + Kind: mapping.GroupVersionKind.Kind, + Name: refName, + }, + MaxReplicas: o.Max, + }, + } + + if o.Min > 0 { + scaler.Spec.MinReplicas = &o.Min + } + + if o.CPUPercent >= 0 { + scaler.Spec.Metrics = []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: &o.CPUPercent, + }, + }, + }, + } + } + + return &scaler +} + +func (o *AutoscaleOptions) createHorizontalPodAutoscalerV1(refName string, mapping *meta.RESTMapping) *autoscalingv1.HorizontalPodAutoscaler { name := o.Name if len(name) == 0 { name = refName diff --git a/pkg/cmd/autoscale/autoscale_test.go b/pkg/cmd/autoscale/autoscale_test.go index 50a2adc8..751edd23 100644 --- a/pkg/cmd/autoscale/autoscale_test.go +++ b/pkg/cmd/autoscale/autoscale_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/assert" autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -87,14 +89,15 @@ func TestAutoscaleValidate(t *testing.T) { } type createHorizontalPodAutoscalerTestCase struct { - name string - options *AutoscaleOptions - refName string - mapping *meta.RESTMapping - expectedHPA *autoscalingv1.HorizontalPodAutoscaler + name string + options *AutoscaleOptions + refName string + mapping *meta.RESTMapping + expectedHPAV2 *autoscalingv2.HorizontalPodAutoscaler + expectedHPAV1 *autoscalingv1.HorizontalPodAutoscaler } -func TestCreateHorizontalPodAutoscaler(t *testing.T) { +func TestCreateHorizontalPodAutoscalerV2(t *testing.T) { tests := []createHorizontalPodAutoscalerTestCase{ { name: "create with all options", @@ -112,7 +115,279 @@ func TestCreateHorizontalPodAutoscaler(t *testing.T) { Kind: "Deployment", }, }, - expectedHPA: &autoscalingv1.HorizontalPodAutoscaler{ + expectedHPAV2: &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-name", + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "deployment-1", + }, + MinReplicas: ptr.To(int32(2)), + MaxReplicas: int32(10), + Metrics: []autoscalingv2.MetricSpec{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricSpec + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#ResourceMetricSource + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricTarget + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To(int32(80)), + }, + }, + }, + }, + }, + }, + }, + { + name: "create without min replicas", + options: &AutoscaleOptions{ + Name: "custom-name-2", + Max: 10, + Min: -1, + CPUPercent: 80, + }, + refName: "deployment-2", + mapping: &meta.RESTMapping{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + }, + expectedHPAV2: &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-name-2", + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "deployment-2", + }, + MinReplicas: nil, + MaxReplicas: int32(10), + Metrics: []autoscalingv2.MetricSpec{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricSpec + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#ResourceMetricSource + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricTarget + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To(int32(80)), + }, + }, + }, + }, + }, + }, + }, + { + name: "create without max replicas", + options: &AutoscaleOptions{ + Name: "custom-name-3", + Max: -1, + Min: 2, + CPUPercent: 80, + }, + refName: "deployment-3", + mapping: &meta.RESTMapping{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + }, + expectedHPAV2: &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-name-3", + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "deployment-3", + }, + MinReplicas: ptr.To(int32(2)), + MaxReplicas: int32(-1), + Metrics: []autoscalingv2.MetricSpec{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricSpec + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#ResourceMetricSource + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricTarget + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To(int32(80)), + }, + }, + }, + }, + }, + }, + }, + { + name: "create without cpu utilization", + options: &AutoscaleOptions{ + Name: "custom-name-4", + Max: 10, + Min: 2, + CPUPercent: -1, + }, + refName: "deployment-4", + mapping: &meta.RESTMapping{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + }, + expectedHPAV2: &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-name-4", + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "deployment-4", + }, + MinReplicas: ptr.To(int32(2)), + MaxReplicas: int32(10), + }, + }, + }, + { + name: "create with replicaset reference", + options: &AutoscaleOptions{ + Name: "replicaset-hpa", + Max: 5, + Min: 1, + CPUPercent: 70, + }, + refName: "frontend", + mapping: &meta.RESTMapping{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "ReplicaSet", + }, + }, + expectedHPAV2: &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "replicaset-hpa", + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "ReplicaSet", + Name: "frontend", + }, + MinReplicas: ptr.To(int32(1)), + MaxReplicas: int32(5), + Metrics: []autoscalingv2.MetricSpec{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricSpec + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#ResourceMetricSource + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricTarget + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To(int32(70)), + }, + }, + }, + }, + }, + }, + }, + { + name: "create with statefulset reference", + options: &AutoscaleOptions{ + Name: "statefulset-hpa", + Max: 8, + Min: 2, + CPUPercent: 60, + }, + refName: "web", + mapping: &meta.RESTMapping{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "StatefulSet", + }, + }, + expectedHPAV2: &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "statefulset-hpa", + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "web", + }, + MinReplicas: ptr.To(int32(2)), + MaxReplicas: int32(8), + Metrics: []autoscalingv2.MetricSpec{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricSpec + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#ResourceMetricSource + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + // Reference: https://pkg.go.dev/k8s.io/api/autoscaling/v2#MetricTarget + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To(int32(60)), + }, + }, + }, + }, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + hpa := tc.options.createHorizontalPodAutoscalerV2(tc.refName, tc.mapping) + assert.Equal(t, tc.expectedHPAV2, hpa) + }) + } +} + +func TestCreateHorizontalPodAutoscalerV1(t *testing.T) { + tests := []createHorizontalPodAutoscalerTestCase{ + { + name: "create with all options", + options: &AutoscaleOptions{ + Name: "custom-name", + Max: 10, + Min: 2, + CPUPercent: 80, + }, + refName: "deployment-1", + mapping: &meta.RESTMapping{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + }, + expectedHPAV1: &autoscalingv1.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-name", }, @@ -144,7 +419,7 @@ func TestCreateHorizontalPodAutoscaler(t *testing.T) { Kind: "Deployment", }, }, - expectedHPA: &autoscalingv1.HorizontalPodAutoscaler{ + expectedHPAV1: &autoscalingv1.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-name-2", }, @@ -176,7 +451,7 @@ func TestCreateHorizontalPodAutoscaler(t *testing.T) { Kind: "Deployment", }, }, - expectedHPA: &autoscalingv1.HorizontalPodAutoscaler{ + expectedHPAV1: &autoscalingv1.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-name-3", }, @@ -208,7 +483,7 @@ func TestCreateHorizontalPodAutoscaler(t *testing.T) { Kind: "Deployment", }, }, - expectedHPA: &autoscalingv1.HorizontalPodAutoscaler{ + expectedHPAV1: &autoscalingv1.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-name-4", }, @@ -218,9 +493,8 @@ func TestCreateHorizontalPodAutoscaler(t *testing.T) { Kind: "Deployment", Name: "deployment-4", }, - MinReplicas: ptr.To(int32(2)), - MaxReplicas: int32(10), - TargetCPUUtilizationPercentage: nil, + MinReplicas: ptr.To(int32(2)), + MaxReplicas: int32(10), }, }, }, @@ -240,7 +514,7 @@ func TestCreateHorizontalPodAutoscaler(t *testing.T) { Kind: "ReplicaSet", }, }, - expectedHPA: &autoscalingv1.HorizontalPodAutoscaler{ + expectedHPAV1: &autoscalingv1.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "replicaset-hpa", }, @@ -272,7 +546,7 @@ func TestCreateHorizontalPodAutoscaler(t *testing.T) { Kind: "StatefulSet", }, }, - expectedHPA: &autoscalingv1.HorizontalPodAutoscaler{ + expectedHPAV1: &autoscalingv1.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "statefulset-hpa", }, @@ -291,8 +565,8 @@ func TestCreateHorizontalPodAutoscaler(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - hpa := tc.options.createHorizontalPodAutoscaler(tc.refName, tc.mapping) - assert.Equal(t, tc.expectedHPA, hpa) + hpa := tc.options.createHorizontalPodAutoscalerV1(tc.refName, tc.mapping) + assert.Equal(t, tc.expectedHPAV1, hpa) }) } }