Merge pull request #3847 from discordianfish/add-horizontal-pod-autoscaler-delay

Automatic merge from submit-queue.

Add HPA up/downscale delay

Still need to test these changes.

docs/cluster_spec.md

@@ -228,9 +228,11 @@ This block contains configurations for the `controller-manager`.
 spec:
   kubeControllerManager:
     horizontalPodAutoscalerSyncPeriod: 15s
+    horizontalPodAutoscalerDownscaleDelay: 5m0s
+    horizontalPodAutoscalerUpscaleDelay: 3m0s
 ```
 
-For more details on `horizontalPodAutoscalerSyncPeriod` see the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/)
+For more details on `horizontalPodAutoscaler` flags see the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/).
 
 ####  Feature Gates
 

pkg/apis/kops/componentconfig.go

@@ -171,7 +171,7 @@ type KubeProxyConfig struct {
 
 // KubeAPIServerConfig defines the configuration for the kube api
 type KubeAPIServerConfig struct {
-	// Image is the docker container usedrun
+	// Image is the docker container used
 	Image string `json:"image,omitempty"`
 	// LogLevel is the logging level of the api
 	LogLevel int32 `json:"logLevel,omitempty" flag:"v" flag-empty:"0"`
@@ -313,8 +313,16 @@ type KubeControllerManagerConfig struct {
 	UseServiceAccountCredentials *bool `json:"useServiceAccountCredentials,omitempty" flag:"use-service-account-credentials"`
 	// HorizontalPodAutoscalerSyncPeriod is the amount of time between syncs
 	// During each period, the controller manager queries the resource utilization
-	// against the metrics specified in each HorizontalPodAutoscaler definition
+	// against the metrics specified in each HorizontalPodAutoscaler definition.
 	HorizontalPodAutoscalerSyncPeriod *metav1.Duration `json:"horizontalPodAutoscalerSyncPeriod,omitempty" flag:"horizontal-pod-autoscaler-sync-period"`
+	// HorizontalPodAutoscalerDownscaleDelay is a duration that specifies
+	// how long the autoscaler has to wait before another downscale
+	// operation can be performed after the current one has completed.
+	HorizontalPodAutoscalerDownscaleDelay *metav1.Duration `json:"horizontalPodAutoscalerDownscaleDelay,omitempty" flag:"horizontal-pod-autoscaler-downscale-delay"`
+	// HorizontalPodAutoscalerUpscaleDelay is a duration that specifies how
+	// long the autoscaler has to wait before another upscale operation can
+	// be performed after the current one has completed.
+	HorizontalPodAutoscalerUpscaleDelay *metav1.Duration `json:"horizontalPodAutoscalerUpscaleDelay,omitempty" flag:"horizontal-pod-autoscaler-downscale-delay"`
 	// FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features.
 	FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"`
 }

pkg/apis/kops/v1alpha1/componentconfig.go

@@ -197,7 +197,7 @@ type KubeAPIServerConfig struct {
 	EtcdCAFile string `json:"etcdCaFile,omitempty" flag:"etcd-cafile"`
 	// EtcdCertFile is the path to a certificate
 	EtcdCertFile string `json:"etcdCertFile,omitempty" flag:"etcd-certfile"`
-	// EtcdKeyFile is the path to a orivate key
+	// EtcdKeyFile is the path to a private key
 	EtcdKeyFile string `json:"etcdKeyFile,omitempty" flag:"etcd-keyfile"`
 	// TODO: Remove unused BasicAuthFile
 	BasicAuthFile string `json:"basicAuthFile,omitempty" flag:"basic-auth-file"`
@@ -313,8 +313,16 @@ type KubeControllerManagerConfig struct {
 	UseServiceAccountCredentials *bool `json:"useServiceAccountCredentials,omitempty" flag:"use-service-account-credentials"`
 	// HorizontalPodAutoscalerSyncPeriod is the amount of time between syncs
 	// During each period, the controller manager queries the resource utilization
-	// against the metrics specified in each HorizontalPodAutoscaler definition
+	// against the metrics specified in each HorizontalPodAutoscaler definition.
 	HorizontalPodAutoscalerSyncPeriod *metav1.Duration `json:"horizontalPodAutoscalerSyncPeriod,omitempty" flag:"horizontal-pod-autoscaler-sync-period"`
+	// HorizontalPodAutoscalerDownscaleDelay is a duration that specifies
+	// how long the autoscaler has to wait before another downscale
+	// operation can be performed after the current one has completed.
+	HorizontalPodAutoscalerDownscaleDelay *metav1.Duration `json:"horizontalPodAutoscalerDownscaleDelay,omitempty" flag:"horizontal-pod-autoscaler-downscale-delay"`
+	// HorizontalPodAutoscalerUpscaleDelay is a duration that specifies how
+	// long the autoscaler has to wait before another upscale operation can
+	// be performed after the current one has completed.
+	HorizontalPodAutoscalerUpscaleDelay *metav1.Duration `json:"horizontalPodAutoscalerUpscaleDelay,omitempty" flag:"horizontal-pod-autoscaler-downscale-delay"`
 	// FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features.
 	FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"`
 }

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -1940,6 +1940,8 @@ func autoConvert_v1alpha1_KubeControllerManagerConfig_To_kops_KubeControllerMana
 	out.TerminatedPodGCThreshold = in.TerminatedPodGCThreshold
 	out.UseServiceAccountCredentials = in.UseServiceAccountCredentials
 	out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod
+	out.HorizontalPodAutoscalerDownscaleDelay = in.HorizontalPodAutoscalerDownscaleDelay
+	out.HorizontalPodAutoscalerUpscaleDelay = in.HorizontalPodAutoscalerUpscaleDelay
 	out.FeatureGates = in.FeatureGates
 	return nil
 }
@@ -1973,6 +1975,8 @@ func autoConvert_kops_KubeControllerManagerConfig_To_v1alpha1_KubeControllerMana
 	out.TerminatedPodGCThreshold = in.TerminatedPodGCThreshold
 	out.UseServiceAccountCredentials = in.UseServiceAccountCredentials
 	out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod
+	out.HorizontalPodAutoscalerDownscaleDelay = in.HorizontalPodAutoscalerDownscaleDelay
+	out.HorizontalPodAutoscalerUpscaleDelay = in.HorizontalPodAutoscalerUpscaleDelay
 	out.FeatureGates = in.FeatureGates
 	return nil
 }

pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go

@@ -2155,6 +2155,24 @@ func (in *KubeControllerManagerConfig) DeepCopyInto(out *KubeControllerManagerCo
 			**out = **in
 		}
 	}
+	if in.HorizontalPodAutoscalerDownscaleDelay != nil {
+		in, out := &in.HorizontalPodAutoscalerDownscaleDelay, &out.HorizontalPodAutoscalerDownscaleDelay
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(v1.Duration)
+			**out = **in
+		}
+	}
+	if in.HorizontalPodAutoscalerUpscaleDelay != nil {
+		in, out := &in.HorizontalPodAutoscalerUpscaleDelay, &out.HorizontalPodAutoscalerUpscaleDelay
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(v1.Duration)
+			**out = **in
+		}
+	}
 	if in.FeatureGates != nil {
 		in, out := &in.FeatureGates, &out.FeatureGates
 		*out = make(map[string]string, len(*in))

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -197,7 +197,7 @@ type KubeAPIServerConfig struct {
 	EtcdCAFile string `json:"etcdCaFile,omitempty" flag:"etcd-cafile"`
 	// EtcdCertFile is the path to a certificate
 	EtcdCertFile string `json:"etcdCertFile,omitempty" flag:"etcd-certfile"`
-	// EtcdKeyFile is the path to a orivate key
+	// EtcdKeyFile is the path to a private key
 	EtcdKeyFile string `json:"etcdKeyFile,omitempty" flag:"etcd-keyfile"`
 	// TODO: Remove unused BasicAuthFile
 	BasicAuthFile string `json:"basicAuthFile,omitempty" flag:"basic-auth-file"`
@@ -313,8 +313,16 @@ type KubeControllerManagerConfig struct {
 	UseServiceAccountCredentials *bool `json:"useServiceAccountCredentials,omitempty" flag:"use-service-account-credentials"`
 	// HorizontalPodAutoscalerSyncPeriod is the amount of time between syncs
 	// During each period, the controller manager queries the resource utilization
-	// against the metrics specified in each HorizontalPodAutoscaler definition
+	// against the metrics specified in each HorizontalPodAutoscaler definition.
 	HorizontalPodAutoscalerSyncPeriod *metav1.Duration `json:"horizontalPodAutoscalerSyncPeriod,omitempty" flag:"horizontal-pod-autoscaler-sync-period"`
+	// HorizontalPodAutoscalerDownscaleDelay is a duration that specifies
+	// how long the autoscaler has to wait before another downscale
+	// operation can be performed after the current one has completed.
+	HorizontalPodAutoscalerDownscaleDelay *metav1.Duration `json:"horizontalPodAutoscalerDownscaleDelay,omitempty" flag:"horizontal-pod-autoscaler-downscale-delay"`
+	// HorizontalPodAutoscalerUpscaleDelay is a duration that specifies how
+	// long the autoscaler has to wait before another upscale operation can
+	// be performed after the current one has completed.
+	HorizontalPodAutoscalerUpscaleDelay *metav1.Duration `json:"horizontalPodAutoscalerUpscaleDelay,omitempty" flag:"horizontal-pod-autoscaler-downscale-delay"`
 	// FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features.
 	FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"`
 }

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -2235,6 +2235,8 @@ func autoConvert_kops_KubeControllerManagerConfig_To_v1alpha2_KubeControllerMana
 	out.TerminatedPodGCThreshold = in.TerminatedPodGCThreshold
 	out.UseServiceAccountCredentials = in.UseServiceAccountCredentials
 	out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod
+	out.HorizontalPodAutoscalerDownscaleDelay = in.HorizontalPodAutoscalerDownscaleDelay
+	out.HorizontalPodAutoscalerUpscaleDelay = in.HorizontalPodAutoscalerUpscaleDelay
 	out.FeatureGates = in.FeatureGates
 	return nil
 }

pkg/apis/kops/zz_generated.deepcopy.go

@@ -2500,6 +2500,24 @@ func (in *KubeControllerManagerConfig) DeepCopyInto(out *KubeControllerManagerCo
 			**out = **in
 		}
 	}
+	if in.HorizontalPodAutoscalerDownscaleDelay != nil {
+		in, out := &in.HorizontalPodAutoscalerDownscaleDelay, &out.HorizontalPodAutoscalerDownscaleDelay
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(v1.Duration)
+			**out = **in
+		}
+	}
+	if in.HorizontalPodAutoscalerUpscaleDelay != nil {
+		in, out := &in.HorizontalPodAutoscalerUpscaleDelay, &out.HorizontalPodAutoscalerUpscaleDelay
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(v1.Duration)
+			**out = **in
+		}
+	}
 	if in.FeatureGates != nil {
 		in, out := &in.FeatureGates, &out.FeatureGates
 		*out = make(map[string]string, len(*in))