diff --git a/cluster-autoscaler/clusterstate/clusterstate.go b/cluster-autoscaler/clusterstate/clusterstate.go index c4ce6d8c61..ee08bbd77e 100644 --- a/cluster-autoscaler/clusterstate/clusterstate.go +++ b/cluster-autoscaler/clusterstate/clusterstate.go @@ -125,7 +125,7 @@ type ClusterStateRegistry struct { incorrectNodeGroupSizes map[string]IncorrectNodeGroupSize unregisteredNodes map[string]UnregisteredNode candidatesForScaleDown map[string][]string - nodeGroupBackoffInfo *backoff.Backoff + nodeGroupBackoffInfo backoff.Backoff lastStatus *api.ClusterAutoscalerStatus lastScaleDownUpdateTime time.Time logRecorder *utils.LogEventRecorder @@ -148,7 +148,7 @@ func NewClusterStateRegistry(cloudProvider cloudprovider.CloudProvider, config C incorrectNodeGroupSizes: make(map[string]IncorrectNodeGroupSize), unregisteredNodes: make(map[string]UnregisteredNode), candidatesForScaleDown: make(map[string][]string), - nodeGroupBackoffInfo: backoff.NewBackoff(InitialNodeGroupBackoffDuration, MaxNodeGroupBackoffDuration, NodeGroupBackoffResetTimeout), + nodeGroupBackoffInfo: backoff.NewExponentialBackoff(InitialNodeGroupBackoffDuration, MaxNodeGroupBackoffDuration, NodeGroupBackoffResetTimeout), lastStatus: emptyStatus, logRecorder: logRecorder, } diff --git a/cluster-autoscaler/utils/backoff/backoff.go b/cluster-autoscaler/utils/backoff/backoff.go index fad0589a27..c9b3b497ab 100644 --- a/cluster-autoscaler/utils/backoff/backoff.go +++ b/cluster-autoscaler/utils/backoff/backoff.go @@ -20,64 +20,14 @@ import ( "time" ) -type backoffInfo struct { - duration time.Duration - backoffUntil time.Time - lastFailedExecution time.Time -} - -// Backoff handles backing off executions. -type Backoff struct { - maxBackoffDuration time.Duration - initialBackoffDuration time.Duration - backoffResetTimeout time.Duration - backoffInfo map[string]backoffInfo -} - -// NewBackoff creates an instance of Backoff. -func NewBackoff(initialBackoffDuration time.Duration, maxBackoffDuration time.Duration, backoffResetTimeout time.Duration) *Backoff { - return &Backoff{maxBackoffDuration, initialBackoffDuration, backoffResetTimeout, make(map[string]backoffInfo)} -} - -// RemoveStaleBackoffData removes stale backoff data. -func (b *Backoff) RemoveStaleBackoffData(currentTime time.Time) { - for key, backoffInfo := range b.backoffInfo { - if backoffInfo.lastFailedExecution.Add(b.backoffResetTimeout).Before(currentTime) { - delete(b.backoffInfo, key) - } - } -} - -// Backoff execution for the given key. Returns time till execution is backed off. -func (b *Backoff) Backoff(key string, currentTime time.Time) time.Time { - duration := b.initialBackoffDuration - if backoffInfo, found := b.backoffInfo[key]; found { - // Multiple concurrent scale-ups failing shouldn't cause backoff - // duration to increase, so we only increase it if we're not in - // backoff right now. - if backoffInfo.backoffUntil.Before(currentTime) { - duration = 2 * backoffInfo.duration - if duration > b.maxBackoffDuration { - duration = b.maxBackoffDuration - } - } - } - backoffUntil := currentTime.Add(duration) - b.backoffInfo[key] = backoffInfo{ - duration: duration, - backoffUntil: backoffUntil, - lastFailedExecution: currentTime, - } - return backoffUntil -} - -// RemoveBackoff removes backoff data for the given key. -func (b *Backoff) RemoveBackoff(key string) { - delete(b.backoffInfo, key) -} - -// IsBackedOff returns true if execution is backed off for the given key. -func (b *Backoff) IsBackedOff(key string, currentTime time.Time) bool { - backoffInfo, found := b.backoffInfo[key] - return found && backoffInfo.backoffUntil.After(currentTime) +// Backoff allows time-based backing off of node groups considered in scale up algorithm +type Backoff interface { + // Backoff execution for the given key. Returns time till execution is backed off. + Backoff(key string, currentTime time.Time) time.Time + // IsBackedOff returns true if execution is backed off for the given key. + IsBackedOff(key string, currentTime time.Time) bool + // RemoveBackoff removes backoff data for the given key. + RemoveBackoff(key string) + // RemoveStaleBackoffData removes stale backoff data. + RemoveStaleBackoffData(currentTime time.Time) } diff --git a/cluster-autoscaler/utils/backoff/exponential_backoff.go b/cluster-autoscaler/utils/backoff/exponential_backoff.go new file mode 100644 index 0000000000..fad5370989 --- /dev/null +++ b/cluster-autoscaler/utils/backoff/exponential_backoff.go @@ -0,0 +1,83 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backoff + +import ( + "time" +) + +// Backoff handles backing off executions. +type exponentialBackoff struct { + maxBackoffDuration time.Duration + initialBackoffDuration time.Duration + backoffResetTimeout time.Duration + backoffInfo map[string]exponentialBackoffInfo +} + +type exponentialBackoffInfo struct { + duration time.Duration + backoffUntil time.Time + lastFailedExecution time.Time +} + +// NewExponentialBackoff creates an instance of exponential backoff. +func NewExponentialBackoff(initialBackoffDuration time.Duration, maxBackoffDuration time.Duration, backoffResetTimeout time.Duration) Backoff { + return &exponentialBackoff{maxBackoffDuration, initialBackoffDuration, backoffResetTimeout, make(map[string]exponentialBackoffInfo)} +} + +// Backoff execution for the given key. Returns time till execution is backed off. +func (b *exponentialBackoff) Backoff(key string, currentTime time.Time) time.Time { + duration := b.initialBackoffDuration + if backoffInfo, found := b.backoffInfo[key]; found { + // Multiple concurrent scale-ups failing shouldn't cause backoff + // duration to increase, so we only increase it if we're not in + // backoff right now. + if backoffInfo.backoffUntil.Before(currentTime) { + duration = 2 * backoffInfo.duration + if duration > b.maxBackoffDuration { + duration = b.maxBackoffDuration + } + } + } + backoffUntil := currentTime.Add(duration) + b.backoffInfo[key] = exponentialBackoffInfo{ + duration: duration, + backoffUntil: backoffUntil, + lastFailedExecution: currentTime, + } + return backoffUntil +} + +// IsBackedOff returns true if execution is backed off for the given key. +func (b *exponentialBackoff) IsBackedOff(key string, currentTime time.Time) bool { + backoffInfo, found := b.backoffInfo[key] + return found && backoffInfo.backoffUntil.After(currentTime) +} + +// RemoveBackoff removes backoff data for the given key. +func (b *exponentialBackoff) RemoveBackoff(key string) { + delete(b.backoffInfo, key) +} + +// RemoveStaleBackoffData removes stale backoff data. +func (b *exponentialBackoff) RemoveStaleBackoffData(currentTime time.Time) { + for key, backoffInfo := range b.backoffInfo { + if backoffInfo.lastFailedExecution.Add(b.backoffResetTimeout).Before(currentTime) { + delete(b.backoffInfo, key) + } + } +} diff --git a/cluster-autoscaler/utils/backoff/backoff_test.go b/cluster-autoscaler/utils/backoff/exponential_backoff_test.go similarity index 82% rename from cluster-autoscaler/utils/backoff/backoff_test.go rename to cluster-autoscaler/utils/backoff/exponential_backoff_test.go index cbaa60286f..e145abecf0 100644 --- a/cluster-autoscaler/utils/backoff/backoff_test.go +++ b/cluster-autoscaler/utils/backoff/exponential_backoff_test.go @@ -24,7 +24,7 @@ import ( ) func TestBackoffTwoKeys(t *testing.T) { - backoff := NewBackoff(10*time.Minute, time.Hour, 3*time.Hour) + backoff := NewExponentialBackoff(10*time.Minute, time.Hour, 3*time.Hour) startTime := time.Now() assert.False(t, backoff.IsBackedOff("key1", startTime)) assert.False(t, backoff.IsBackedOff("key2", startTime)) @@ -35,7 +35,7 @@ func TestBackoffTwoKeys(t *testing.T) { } func TestMaxBackoff(t *testing.T) { - backoff := NewBackoff(1*time.Minute, 3*time.Minute, 3*time.Hour) + backoff := NewExponentialBackoff(1*time.Minute, 3*time.Minute, 3*time.Hour) startTime := time.Now() backoff.Backoff("key1", startTime) assert.True(t, backoff.IsBackedOff("key1", startTime)) @@ -49,7 +49,7 @@ func TestMaxBackoff(t *testing.T) { } func TestRemoveBackoff(t *testing.T) { - backoff := NewBackoff(1*time.Minute, 3*time.Minute, 3*time.Hour) + backoff := NewExponentialBackoff(1*time.Minute, 3*time.Minute, 3*time.Hour) startTime := time.Now() backoff.Backoff("key1", startTime) assert.True(t, backoff.IsBackedOff("key1", startTime)) @@ -58,14 +58,14 @@ func TestRemoveBackoff(t *testing.T) { } func TestResetStaleBackoffData(t *testing.T) { - backoff := NewBackoff(1*time.Minute, 3*time.Minute, 3*time.Hour) + backoff := NewExponentialBackoff(1*time.Minute, 3*time.Minute, 3*time.Hour) startTime := time.Now() backoff.Backoff("key1", startTime) backoff.Backoff("key2", startTime.Add(time.Hour)) backoff.RemoveStaleBackoffData(startTime.Add(time.Hour)) - assert.Equal(t, 2, len(backoff.backoffInfo)) + assert.Equal(t, 2, len(backoff.(*exponentialBackoff).backoffInfo)) backoff.RemoveStaleBackoffData(startTime.Add(4 * time.Hour)) - assert.Equal(t, 1, len(backoff.backoffInfo)) + assert.Equal(t, 1, len(backoff.(*exponentialBackoff).backoffInfo)) backoff.RemoveStaleBackoffData(startTime.Add(5 * time.Hour)) - assert.Equal(t, 0, len(backoff.backoffInfo)) + assert.Equal(t, 0, len(backoff.(*exponentialBackoff).backoffInfo)) }