111 lines
4.0 KiB
Go
111 lines
4.0 KiB
Go
/*
|
|
Copyright 2018 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package backoff
|
|
|
|
import (
|
|
"time"
|
|
|
|
"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
|
|
|
|
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
|
|
)
|
|
|
|
// Backoff handles backing off executions.
|
|
type exponentialBackoff struct {
|
|
maxBackoffDuration time.Duration
|
|
initialBackoffDuration time.Duration
|
|
backoffResetTimeout time.Duration
|
|
backoffInfo map[string]exponentialBackoffInfo
|
|
nodeGroupKey func(nodeGroup cloudprovider.NodeGroup) string
|
|
}
|
|
|
|
type exponentialBackoffInfo struct {
|
|
duration time.Duration
|
|
backoffUntil time.Time
|
|
lastFailedExecution time.Time
|
|
}
|
|
|
|
// NewExponentialBackoff creates an instance of exponential backoff.
|
|
func NewExponentialBackoff(
|
|
initialBackoffDuration time.Duration,
|
|
maxBackoffDuration time.Duration,
|
|
backoffResetTimeout time.Duration,
|
|
nodeGroupKey func(nodeGroup cloudprovider.NodeGroup) string) Backoff {
|
|
return &exponentialBackoff{
|
|
maxBackoffDuration: maxBackoffDuration,
|
|
initialBackoffDuration: initialBackoffDuration,
|
|
backoffResetTimeout: backoffResetTimeout,
|
|
backoffInfo: make(map[string]exponentialBackoffInfo),
|
|
nodeGroupKey: nodeGroupKey,
|
|
}
|
|
}
|
|
|
|
// NewIdBasedExponentialBackoff creates an instance of exponential backoff with node group Id used as a key.
|
|
func NewIdBasedExponentialBackoff(initialBackoffDuration time.Duration, maxBackoffDuration time.Duration, backoffResetTimeout time.Duration) Backoff {
|
|
return NewExponentialBackoff(
|
|
initialBackoffDuration,
|
|
maxBackoffDuration,
|
|
backoffResetTimeout,
|
|
func(nodeGroup cloudprovider.NodeGroup) string {
|
|
return nodeGroup.Id()
|
|
})
|
|
}
|
|
|
|
// Backoff execution for the given node group. Returns time till execution is backed off.
|
|
func (b *exponentialBackoff) Backoff(nodeGroup cloudprovider.NodeGroup, nodeInfo *schedulernodeinfo.NodeInfo, errorClass cloudprovider.InstanceErrorClass, errorCode string, currentTime time.Time) time.Time {
|
|
duration := b.initialBackoffDuration
|
|
key := b.nodeGroupKey(nodeGroup)
|
|
if backoffInfo, found := b.backoffInfo[key]; found {
|
|
// Multiple concurrent scale-ups failing shouldn't cause backoff
|
|
// duration to increase, so we only increase it if we're not in
|
|
// backoff right now.
|
|
if backoffInfo.backoffUntil.Before(currentTime) {
|
|
duration = 2 * backoffInfo.duration
|
|
if duration > b.maxBackoffDuration {
|
|
duration = b.maxBackoffDuration
|
|
}
|
|
}
|
|
}
|
|
backoffUntil := currentTime.Add(duration)
|
|
b.backoffInfo[key] = exponentialBackoffInfo{
|
|
duration: duration,
|
|
backoffUntil: backoffUntil,
|
|
lastFailedExecution: currentTime,
|
|
}
|
|
return backoffUntil
|
|
}
|
|
|
|
// IsBackedOff returns true if execution is backed off for the given node group.
|
|
func (b *exponentialBackoff) IsBackedOff(nodeGroup cloudprovider.NodeGroup, nodeInfo *schedulernodeinfo.NodeInfo, currentTime time.Time) bool {
|
|
backoffInfo, found := b.backoffInfo[b.nodeGroupKey(nodeGroup)]
|
|
return found && backoffInfo.backoffUntil.After(currentTime)
|
|
}
|
|
|
|
// RemoveBackoff removes backoff data for the given node group.
|
|
func (b *exponentialBackoff) RemoveBackoff(nodeGroup cloudprovider.NodeGroup, nodeInfo *schedulernodeinfo.NodeInfo) {
|
|
delete(b.backoffInfo, b.nodeGroupKey(nodeGroup))
|
|
}
|
|
|
|
// RemoveStaleBackoffData removes stale backoff data.
|
|
func (b *exponentialBackoff) RemoveStaleBackoffData(currentTime time.Time) {
|
|
for key, backoffInfo := range b.backoffInfo {
|
|
if backoffInfo.lastFailedExecution.Add(b.backoffResetTimeout).Before(currentTime) {
|
|
delete(b.backoffInfo, key)
|
|
}
|
|
}
|
|
}
|