Enforce provisioning requests processing even if all pods are new
This commit is contained in:
parent
a587c552af
commit
d7c325abf7
|
@ -520,13 +520,15 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
forceScaleUp := a.processors.ScaleUpEnforcer.ShouldForceScaleUp(unschedulablePodsToHelp)
|
||||||
|
|
||||||
if len(unschedulablePodsToHelp) == 0 {
|
if len(unschedulablePodsToHelp) == 0 {
|
||||||
scaleUpStatus.Result = status.ScaleUpNotNeeded
|
scaleUpStatus.Result = status.ScaleUpNotNeeded
|
||||||
klog.V(1).Info("No unschedulable pods")
|
klog.V(1).Info("No unschedulable pods")
|
||||||
} else if a.MaxNodesTotal > 0 && len(readyNodes) >= a.MaxNodesTotal {
|
} else if a.MaxNodesTotal > 0 && len(readyNodes) >= a.MaxNodesTotal && !forceScaleUp {
|
||||||
scaleUpStatus.Result = status.ScaleUpNoOptionsAvailable
|
scaleUpStatus.Result = status.ScaleUpNoOptionsAvailable
|
||||||
klog.V(1).Infof("Max total nodes in cluster reached: %v. Current number of ready nodes: %v", a.MaxNodesTotal, len(readyNodes))
|
klog.V(1).Infof("Max total nodes in cluster reached: %v. Current number of ready nodes: %v", a.MaxNodesTotal, len(readyNodes))
|
||||||
} else if len(a.BypassedSchedulers) == 0 && allPodsAreNew(unschedulablePodsToHelp, currentTime) {
|
} else if len(a.BypassedSchedulers) == 0 && !forceScaleUp && allPodsAreNew(unschedulablePodsToHelp, currentTime) {
|
||||||
// The assumption here is that these pods have been created very recently and probably there
|
// The assumption here is that these pods have been created very recently and probably there
|
||||||
// is more pods to come. In theory we could check the newest pod time but then if pod were created
|
// is more pods to come. In theory we could check the newest pod time but then if pod were created
|
||||||
// slowly but at the pace of 1 every 2 seconds then no scale up would be triggered for long time.
|
// slowly but at the pace of 1 every 2 seconds then no scale up would be triggered for long time.
|
||||||
|
|
|
@ -563,6 +563,8 @@ func buildAutoscaler(context ctx.Context, debuggingSnapshotter debuggingsnapshot
|
||||||
opts.LoopStartNotifier = loopstart.NewObserversList([]loopstart.Observer{provreqProcesor})
|
opts.LoopStartNotifier = loopstart.NewObserversList([]loopstart.Observer{provreqProcesor})
|
||||||
|
|
||||||
podListProcessor.AddProcessor(provreqProcesor)
|
podListProcessor.AddProcessor(provreqProcesor)
|
||||||
|
|
||||||
|
opts.Processors.ScaleUpEnforcer = provreq.NewProvisioningRequestScaleUpEnforcer()
|
||||||
}
|
}
|
||||||
|
|
||||||
if *proactiveScaleupEnabled {
|
if *proactiveScaleupEnabled {
|
||||||
|
|
|
@ -0,0 +1,38 @@
|
||||||
|
/*
|
||||||
|
Copyright 2025 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package pods
|
||||||
|
|
||||||
|
import apiv1 "k8s.io/api/core/v1"
|
||||||
|
|
||||||
|
// ScaleUpEnforcer can force scale up even if all pods are new or MaxNodesTotal was achieved.
|
||||||
|
type ScaleUpEnforcer interface {
|
||||||
|
ShouldForceScaleUp(unschedulablePods []*apiv1.Pod) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NoOpScaleUpEnforcer returns false by default in case of ProvisioningRequests disabled.
|
||||||
|
type NoOpScaleUpEnforcer struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDefaultScaleUpEnforcer creates an instance of ScaleUpEnforcer.
|
||||||
|
func NewDefaultScaleUpEnforcer() ScaleUpEnforcer {
|
||||||
|
return &NoOpScaleUpEnforcer{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldForceScaleUp returns false by default.
|
||||||
|
func (p *NoOpScaleUpEnforcer) ShouldForceScaleUp(unschedulablePods []*apiv1.Pod) bool {
|
||||||
|
return false
|
||||||
|
}
|
|
@ -0,0 +1,34 @@
|
||||||
|
/*
|
||||||
|
Copyright 2025 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package pods
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
apiv1 "k8s.io/api/core/v1"
|
||||||
|
testutils "k8s.io/autoscaler/cluster-autoscaler/utils/test"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDefaultScaleUpEnforcer(t *testing.T) {
|
||||||
|
p1 := testutils.BuildTestPod("p1", 40, 0)
|
||||||
|
unschedulablePods := []*apiv1.Pod{p1}
|
||||||
|
scaleUpEnforcer := NewDefaultScaleUpEnforcer()
|
||||||
|
forceScaleUp := scaleUpEnforcer.ShouldForceScaleUp(unschedulablePods)
|
||||||
|
if forceScaleUp {
|
||||||
|
t.Errorf("Error: scaleUpEnforcer should not force scale up by default")
|
||||||
|
}
|
||||||
|
}
|
|
@ -74,6 +74,8 @@ type AutoscalingProcessors struct {
|
||||||
ScaleStateNotifier *nodegroupchange.NodeGroupChangeObserversList
|
ScaleStateNotifier *nodegroupchange.NodeGroupChangeObserversList
|
||||||
// AsyncNodeGroupChecker checks if node group is upcoming or not
|
// AsyncNodeGroupChecker checks if node group is upcoming or not
|
||||||
AsyncNodeGroupStateChecker asyncnodegroups.AsyncNodeGroupStateChecker
|
AsyncNodeGroupStateChecker asyncnodegroups.AsyncNodeGroupStateChecker
|
||||||
|
// ScaleUpEnforcer can force scale up even if all pods are new or MaxNodesTotal was achieved.
|
||||||
|
ScaleUpEnforcer pods.ScaleUpEnforcer
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultProcessors returns default set of processors.
|
// DefaultProcessors returns default set of processors.
|
||||||
|
@ -100,6 +102,7 @@ func DefaultProcessors(options config.AutoscalingOptions) *AutoscalingProcessors
|
||||||
TemplateNodeInfoProvider: nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false),
|
TemplateNodeInfoProvider: nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false),
|
||||||
ScaleDownCandidatesNotifier: scaledowncandidates.NewObserversList(),
|
ScaleDownCandidatesNotifier: scaledowncandidates.NewObserversList(),
|
||||||
ScaleStateNotifier: nodegroupchange.NewNodeGroupChangeObserversList(),
|
ScaleStateNotifier: nodegroupchange.NewNodeGroupChangeObserversList(),
|
||||||
|
ScaleUpEnforcer: pods.NewDefaultScaleUpEnforcer(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,7 @@ import (
|
||||||
|
|
||||||
apiv1 "k8s.io/api/core/v1"
|
apiv1 "k8s.io/api/core/v1"
|
||||||
corev1 "k8s.io/api/core/v1"
|
corev1 "k8s.io/api/core/v1"
|
||||||
"k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
|
v1 "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
|
||||||
"k8s.io/autoscaler/cluster-autoscaler/context"
|
"k8s.io/autoscaler/cluster-autoscaler/context"
|
||||||
"k8s.io/autoscaler/cluster-autoscaler/processors/pods"
|
"k8s.io/autoscaler/cluster-autoscaler/processors/pods"
|
||||||
provreqpods "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/pods"
|
provreqpods "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/pods"
|
||||||
|
|
|
@ -0,0 +1,41 @@
|
||||||
|
/*
|
||||||
|
Copyright 2025 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package provreq
|
||||||
|
|
||||||
|
import (
|
||||||
|
apiv1 "k8s.io/api/core/v1"
|
||||||
|
"k8s.io/autoscaler/cluster-autoscaler/processors/pods"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProvisioningRequestScaleUpEnforcer forces scale up if there is any unschedulable pod that belongs to ProvisioningRequest.
|
||||||
|
type ProvisioningRequestScaleUpEnforcer struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewProvisioningRequestScaleUpEnforcer creates a ProvisioningRequest scale up enforcer.
|
||||||
|
func NewProvisioningRequestScaleUpEnforcer() pods.ScaleUpEnforcer {
|
||||||
|
return &ProvisioningRequestScaleUpEnforcer{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldForceScaleUp forces scale up if there is any unschedulable pod that belongs to ProvisioningRequest.
|
||||||
|
func (p *ProvisioningRequestScaleUpEnforcer) ShouldForceScaleUp(unschedulablePods []*apiv1.Pod) bool {
|
||||||
|
for _, pod := range unschedulablePods {
|
||||||
|
if _, ok := provisioningRequestName(pod); ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
|
@ -0,0 +1,66 @@
|
||||||
|
/*
|
||||||
|
Copyright 2025 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package provreq
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
apiv1 "k8s.io/api/core/v1"
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
v1 "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
|
||||||
|
"k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/pods"
|
||||||
|
testutils "k8s.io/autoscaler/cluster-autoscaler/utils/test"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProvisioningRequestScaleUpEnforcer(t *testing.T) {
|
||||||
|
prPod1 := testutils.BuildTestPod("pr-pod-1", 500, 10)
|
||||||
|
prPod1.Annotations[v1.ProvisioningRequestPodAnnotationKey] = "pr-class"
|
||||||
|
|
||||||
|
prPod2 := testutils.BuildTestPod("pr-pod-2", 500, 10)
|
||||||
|
prPod2.Annotations[pods.DeprecatedProvisioningRequestPodAnnotationKey] = "pr-class-2"
|
||||||
|
|
||||||
|
pod1 := testutils.BuildTestPod("pod-1", 500, 10)
|
||||||
|
pod2 := testutils.BuildTestPod("pod-2", 500, 10)
|
||||||
|
|
||||||
|
testCases := map[string]struct {
|
||||||
|
unschedulablePods []*apiv1.Pod
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
"Any pod with ProvisioningRequest annotation key forces scale up": {
|
||||||
|
unschedulablePods: []*corev1.Pod{prPod1, pod1},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
"Any pod with ProvisioningRequest deprecated annotation key forces scale up": {
|
||||||
|
unschedulablePods: []*corev1.Pod{prPod2, pod1},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
"Pod without ProvisioningRequest annotation key don't force scale up": {
|
||||||
|
unschedulablePods: []*corev1.Pod{pod1, pod2},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
"No pods don't force scale up": {
|
||||||
|
unschedulablePods: []*corev1.Pod{},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range testCases {
|
||||||
|
scaleUpEnforcer := NewProvisioningRequestScaleUpEnforcer()
|
||||||
|
got := scaleUpEnforcer.ShouldForceScaleUp(test.unschedulablePods)
|
||||||
|
assert.Equal(t, got, test.want)
|
||||||
|
}
|
||||||
|
}
|
|
@ -31,6 +31,7 @@ import (
|
||||||
"k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset"
|
"k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset"
|
||||||
"k8s.io/autoscaler/cluster-autoscaler/processors/nodeinfosprovider"
|
"k8s.io/autoscaler/cluster-autoscaler/processors/nodeinfosprovider"
|
||||||
"k8s.io/autoscaler/cluster-autoscaler/processors/nodes"
|
"k8s.io/autoscaler/cluster-autoscaler/processors/nodes"
|
||||||
|
"k8s.io/autoscaler/cluster-autoscaler/processors/pods"
|
||||||
"k8s.io/autoscaler/cluster-autoscaler/processors/scaledowncandidates"
|
"k8s.io/autoscaler/cluster-autoscaler/processors/scaledowncandidates"
|
||||||
"k8s.io/autoscaler/cluster-autoscaler/processors/status"
|
"k8s.io/autoscaler/cluster-autoscaler/processors/status"
|
||||||
"k8s.io/autoscaler/cluster-autoscaler/simulator/scheduling"
|
"k8s.io/autoscaler/cluster-autoscaler/simulator/scheduling"
|
||||||
|
@ -56,5 +57,6 @@ func NewTestProcessors(context *context.AutoscalingContext) *processors.Autoscal
|
||||||
ScaleDownCandidatesNotifier: scaledowncandidates.NewObserversList(),
|
ScaleDownCandidatesNotifier: scaledowncandidates.NewObserversList(),
|
||||||
ScaleStateNotifier: nodegroupchange.NewNodeGroupChangeObserversList(),
|
ScaleStateNotifier: nodegroupchange.NewNodeGroupChangeObserversList(),
|
||||||
AsyncNodeGroupStateChecker: asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker(),
|
AsyncNodeGroupStateChecker: asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker(),
|
||||||
|
ScaleUpEnforcer: pods.NewDefaultScaleUpEnforcer(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue