autoscaler/cluster-autoscaler/cloudprovider/aws/managed_nodegroup_cache_tes...

589 lines
19 KiB
Go

/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"errors"
"strconv"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
apiv1 "k8s.io/api/core/v1"
"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/eks"
"k8s.io/client-go/tools/cache"
test_clock "k8s.io/utils/clock/testing"
)
func TestManagedNodegroupCache(t *testing.T) {
nodegroupName := "nodegroupName"
clusterName := "clusterName"
labelKey := "label key 1"
labelValue := "label value 1"
taintEffect := "effect 1"
taintKey := "key 1"
taintValue := "value 1"
taint := apiv1.Taint{
Effect: apiv1.TaintEffect(taintEffect),
Key: taintKey,
Value: taintValue,
}
c := newManagedNodeGroupCache(nil)
err := c.Add(managedNodegroupCachedObject{
name: nodegroupName,
clusterName: clusterName,
taints: []apiv1.Taint{taint},
labels: map[string]string{labelKey: labelValue},
})
require.NoError(t, err)
obj, ok, err := c.GetByKey(nodegroupName)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, nodegroupName, obj.(managedNodegroupCachedObject).name)
assert.Equal(t, clusterName, obj.(managedNodegroupCachedObject).clusterName)
assert.Equal(t, len(obj.(managedNodegroupCachedObject).labels), 1)
assert.Equal(t, labelValue, obj.(managedNodegroupCachedObject).labels[labelKey])
assert.Equal(t, len(obj.(managedNodegroupCachedObject).taints), 1)
assert.Equal(t, apiv1.TaintEffect(taintEffect), obj.(managedNodegroupCachedObject).taints[0].Effect)
assert.Equal(t, taintKey, obj.(managedNodegroupCachedObject).taints[0].Key)
assert.Equal(t, taintValue, obj.(managedNodegroupCachedObject).taints[0].Value)
}
func TestGetManagedNodegroupWithError(t *testing.T) {
k := &eksMock{}
nodegroupName := "testNodegroup"
clusterName := "testCluster"
k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
}).Return(nil, errors.New("AccessDenied"))
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
// Make sure there's an error but no cache object returned
_, err := c.getManagedNodegroup(nodegroupName, clusterName)
require.Error(t, err)
// Make sure an empty cache object was saved
obj, ok, err := c.GetByKey(nodegroupName)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, nodegroupName, obj.(managedNodegroupCachedObject).name)
assert.Equal(t, clusterName, obj.(managedNodegroupCachedObject).clusterName)
assert.Nil(t, obj.(managedNodegroupCachedObject).labels)
assert.Nil(t, obj.(managedNodegroupCachedObject).taints)
}
func TestGetManagedNodegroupNoTaintsOrLabels(t *testing.T) {
k := &eksMock{}
nodegroupName := "testNodegroup"
clusterName := "testCluster"
amiType := "testAmiType"
capacityType := "testCapacityType"
k8sVersion := "1.19"
// Create test nodegroup
testNodegroup := eks.Nodegroup{
AmiType: &amiType,
ClusterName: &clusterName,
DiskSize: nil,
Labels: nil,
NodegroupName: &nodegroupName,
CapacityType: &capacityType,
Version: &k8sVersion,
Taints: nil,
}
k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil)
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
cacheObj, err := c.getManagedNodegroup(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, cacheObj.name, nodegroupName)
assert.Equal(t, cacheObj.clusterName, clusterName)
assert.Equal(t, len(cacheObj.taints), 0)
assert.Equal(t, len(cacheObj.labels), 4)
assert.Equal(t, cacheObj.labels["amiType"], amiType)
assert.Equal(t, cacheObj.labels["capacityType"], capacityType)
assert.Equal(t, cacheObj.labels["k8sVersion"], k8sVersion)
assert.Equal(t, cacheObj.labels["eks.amazonaws.com/nodegroup"], nodegroupName)
}
func TestGetManagedNodegroupWithTaintsAndLabels(t *testing.T) {
k := &eksMock{}
amiType := "testAmiType"
capacityType := "testCapacityType"
k8sVersion := "1.19"
diskSize := int64(100)
nodegroupName := "testNodegroup"
clusterName := "testCluster"
labelKey1 := "labelKey 1"
labelKey2 := "labelKey 2"
labelValue1 := "testValue 1"
labelValue2 := "testValue 2"
taintEffect1 := "effect 1"
taintKey1 := "key 1"
taintValue1 := "value 1"
taint1 := eks.Taint{
Effect: &taintEffect1,
Key: &taintKey1,
Value: &taintValue1,
}
taintEffect2 := "effect 2"
taintKey2 := "key 2"
taintValue2 := "value 2"
taint2 := eks.Taint{
Effect: &taintEffect2,
Key: &taintKey2,
Value: &taintValue2,
}
// Create test nodegroup
testNodegroup := eks.Nodegroup{
AmiType: &amiType,
ClusterName: &clusterName,
DiskSize: &diskSize,
Labels: map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2},
NodegroupName: &nodegroupName,
CapacityType: &capacityType,
Version: &k8sVersion,
Taints: []*eks.Taint{&taint1, &taint2},
}
k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil)
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
cacheObj, err := c.getManagedNodegroup(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, cacheObj.name, nodegroupName)
assert.Equal(t, cacheObj.clusterName, clusterName)
assert.Equal(t, len(cacheObj.taints), 2)
assert.Equal(t, cacheObj.taints[0].Effect, apiv1.TaintEffect(taintEffect1))
assert.Equal(t, cacheObj.taints[0].Key, taintKey1)
assert.Equal(t, cacheObj.taints[0].Value, taintValue1)
assert.Equal(t, cacheObj.taints[1].Effect, apiv1.TaintEffect(taintEffect2))
assert.Equal(t, cacheObj.taints[1].Key, taintKey2)
assert.Equal(t, cacheObj.taints[1].Value, taintValue2)
assert.Equal(t, len(cacheObj.labels), 7)
assert.Equal(t, cacheObj.labels[labelKey1], labelValue1)
assert.Equal(t, cacheObj.labels[labelKey2], labelValue2)
assert.Equal(t, cacheObj.labels["diskSize"], strconv.FormatInt(diskSize, 10))
assert.Equal(t, cacheObj.labels["amiType"], amiType)
assert.Equal(t, cacheObj.labels["capacityType"], capacityType)
assert.Equal(t, cacheObj.labels["k8sVersion"], k8sVersion)
assert.Equal(t, cacheObj.labels["eks.amazonaws.com/nodegroup"], nodegroupName)
}
func TestGetManagedNodegroupInfoObjectWithError(t *testing.T) {
k := &eksMock{}
nodegroupName := "testNodegroup"
clusterName := "testCluster"
k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
}).Return(nil, errors.New("AccessDenied"))
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
// Make sure there's an error
mngInfoObject, err := c.getManagedNodegroupInfoObject(nodegroupName, clusterName)
require.Error(t, err)
// Make sure an object isn't returned
assert.Nil(t, mngInfoObject)
}
func TestGetManagedNodegroupInfoObjectWithCachedNodegroup(t *testing.T) {
k := &eksMock{}
nodegroupName := "nodegroupName"
clusterName := "clusterName"
labelKey := "label key 1"
labelValue := "label value 1"
taintEffect := "effect 1"
taintKey := "key 1"
taintValue := "value 1"
taint := apiv1.Taint{
Effect: apiv1.TaintEffect(taintEffect),
Key: taintKey,
Value: taintValue,
}
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
err := c.Add(managedNodegroupCachedObject{
name: nodegroupName,
clusterName: clusterName,
taints: []apiv1.Taint{taint},
labels: map[string]string{labelKey: labelValue},
})
mngInfoObject, err := c.getManagedNodegroupInfoObject(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, len(mngInfoObject.labels), 1)
assert.Equal(t, mngInfoObject.labels[labelKey], labelValue)
k.AssertNotCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
})
}
func TestGetManagedNodegroupInfoObjectNoCachedNodegroup(t *testing.T) {
k := &eksMock{}
nodegroupName := "testNodegroup"
clusterName := "testCluster"
amiType := "testAmiType"
capacityType := "testCapacityType"
k8sVersion := "1.19"
diskSize := int64(100)
labelKey1 := "labelKey 1"
labelKey2 := "labelKey 2"
labelValue1 := "testValue 1"
labelValue2 := "testValue 2"
// Create test nodegroup
testNodegroup := eks.Nodegroup{
AmiType: &amiType,
ClusterName: &clusterName,
DiskSize: &diskSize,
Labels: map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2},
NodegroupName: &nodegroupName,
CapacityType: &capacityType,
Version: &k8sVersion,
Taints: nil,
}
k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil)
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
mngInfoObject, err := c.getManagedNodegroupInfoObject(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, len(mngInfoObject.labels), 7)
assert.Equal(t, mngInfoObject.labels[labelKey1], labelValue1)
assert.Equal(t, mngInfoObject.labels[labelKey2], labelValue2)
assert.Equal(t, mngInfoObject.labels["diskSize"], strconv.FormatInt(diskSize, 10))
assert.Equal(t, mngInfoObject.labels["amiType"], amiType)
assert.Equal(t, mngInfoObject.labels["capacityType"], capacityType)
assert.Equal(t, mngInfoObject.labels["k8sVersion"], k8sVersion)
assert.Equal(t, mngInfoObject.labels["eks.amazonaws.com/nodegroup"], nodegroupName)
k.AssertCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
})
}
func TestGetManagedNodegroupLabelsWithCachedNodegroup(t *testing.T) {
k := &eksMock{}
nodegroupName := "nodegroupName"
clusterName := "clusterName"
labelKey := "label key 1"
labelValue := "label value 1"
taintEffect := "effect 1"
taintKey := "key 1"
taintValue := "value 1"
taint := apiv1.Taint{
Effect: apiv1.TaintEffect(taintEffect),
Key: taintKey,
Value: taintValue,
}
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
err := c.Add(managedNodegroupCachedObject{
name: nodegroupName,
clusterName: clusterName,
taints: []apiv1.Taint{taint},
labels: map[string]string{labelKey: labelValue},
})
labelsMap, err := c.getManagedNodegroupLabels(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, len(labelsMap), 1)
assert.Equal(t, labelsMap[labelKey], labelValue)
k.AssertNotCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
})
}
func TestGetManagedNodegroupLabelsNoCachedNodegroup(t *testing.T) {
k := &eksMock{}
nodegroupName := "testNodegroup"
clusterName := "testCluster"
amiType := "testAmiType"
capacityType := "testCapacityType"
k8sVersion := "1.19"
diskSize := int64(100)
labelKey1 := "labelKey 1"
labelKey2 := "labelKey 2"
labelValue1 := "testValue 1"
labelValue2 := "testValue 2"
// Create test nodegroup
testNodegroup := eks.Nodegroup{
AmiType: &amiType,
ClusterName: &clusterName,
DiskSize: &diskSize,
Labels: map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2},
NodegroupName: &nodegroupName,
CapacityType: &capacityType,
Version: &k8sVersion,
Taints: nil,
}
k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil)
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
labelsMap, err := c.getManagedNodegroupLabels(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, len(labelsMap), 7)
assert.Equal(t, labelsMap[labelKey1], labelValue1)
assert.Equal(t, labelsMap[labelKey2], labelValue2)
assert.Equal(t, labelsMap["diskSize"], strconv.FormatInt(diskSize, 10))
assert.Equal(t, labelsMap["amiType"], amiType)
assert.Equal(t, labelsMap["capacityType"], capacityType)
assert.Equal(t, labelsMap["k8sVersion"], k8sVersion)
assert.Equal(t, labelsMap["eks.amazonaws.com/nodegroup"], nodegroupName)
k.AssertCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
})
}
func TestGetManagedNodegroupLabelsWithCachedNodegroupThatExpires(t *testing.T) {
k := &eksMock{}
nodegroupName := "testNodegroup"
clusterName := "testCluster"
amiType := "testAmiType"
capacityType := "testCapacityType"
k8sVersion := "1.19"
diskSize := int64(100)
labelKey1 := "labelKey 1"
labelKey2 := "labelKey 2"
labelValue1 := "testValue 1"
labelValue2 := "testValue 2"
// Create test nodegroup
testNodegroup := eks.Nodegroup{
AmiType: &amiType,
ClusterName: &clusterName,
DiskSize: &diskSize,
Labels: map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2},
NodegroupName: &nodegroupName,
CapacityType: &capacityType,
Version: &k8sVersion,
Taints: nil,
}
k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil)
fakeClock := test_clock.NewFakeClock(time.Unix(0, 0))
fakeStore := cache.NewFakeExpirationStore(
func(obj interface{}) (s string, e error) {
return obj.(managedNodegroupCachedObject).name, nil
},
nil,
&cache.TTLPolicy{
TTL: managedNodegroupCachedTTL,
Clock: fakeClock,
},
fakeClock,
)
// Create cache with fake clock
c := newManagedNodeGroupCacheWithClock(&awsWrapper{nil, nil, k}, fakeClock, fakeStore)
// Add nodegroup entry that will expire
err := c.Add(managedNodegroupCachedObject{
name: nodegroupName,
clusterName: clusterName,
taints: make([]apiv1.Taint, 0),
labels: map[string]string{labelKey1: labelValue1},
})
require.NoError(t, err)
obj, ok, err := c.GetByKey(nodegroupName)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, nodegroupName, obj.(managedNodegroupCachedObject).name)
assert.Equal(t, clusterName, obj.(managedNodegroupCachedObject).clusterName)
assert.Equal(t, len(obj.(managedNodegroupCachedObject).labels), 1)
assert.Equal(t, labelValue1, obj.(managedNodegroupCachedObject).labels[labelKey1])
assert.Equal(t, len(obj.(managedNodegroupCachedObject).taints), 0)
// Query for nodegroup entry before it expires
labelsMap, err := c.getManagedNodegroupLabels(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, len(labelsMap), 1)
assert.Equal(t, labelsMap[labelKey1], labelValue1)
k.AssertNotCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
})
// Expire nodegroup
fakeClock.SetTime(time.Unix(0, 0).Add(managedNodegroupCachedTTL + 1*time.Minute))
// Query for nodegroup entry after it expires - should have the new labels added
newLabelsMap, err := c.getManagedNodegroupLabels(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, len(newLabelsMap), 7)
assert.Equal(t, newLabelsMap[labelKey1], labelValue1)
assert.Equal(t, newLabelsMap[labelKey2], labelValue2)
assert.Equal(t, newLabelsMap["diskSize"], strconv.FormatInt(diskSize, 10))
assert.Equal(t, newLabelsMap["amiType"], amiType)
assert.Equal(t, newLabelsMap["capacityType"], capacityType)
assert.Equal(t, newLabelsMap["k8sVersion"], k8sVersion)
assert.Equal(t, newLabelsMap["eks.amazonaws.com/nodegroup"], nodegroupName)
k.AssertCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
})
}
func TestGetManagedNodegroupTaintsWithCachedNodegroup(t *testing.T) {
k := &eksMock{}
nodegroupName := "nodegroupName"
clusterName := "clusterName"
labelKey := "label key 1"
labelValue := "label value 1"
taintEffect := "effect 1"
taintKey := "key 1"
taintValue := "value 1"
taint := apiv1.Taint{
Effect: apiv1.TaintEffect(taintEffect),
Key: taintKey,
Value: taintValue,
}
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
err := c.Add(managedNodegroupCachedObject{
name: nodegroupName,
clusterName: clusterName,
taints: []apiv1.Taint{taint},
labels: map[string]string{labelKey: labelValue},
})
taintsList, err := c.getManagedNodegroupTaints(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, len(taintsList), 1)
assert.Equal(t, taintsList[0].Effect, apiv1.TaintEffect(taintEffect))
assert.Equal(t, taintsList[0].Key, taintKey)
assert.Equal(t, taintsList[0].Value, taintValue)
k.AssertNotCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
})
}
func TestGetManagedNodegroupTaintsNoCachedNodegroup(t *testing.T) {
k := &eksMock{}
nodegroupName := "testNodegroup"
clusterName := "testCluster"
amiType := "testAmiType"
capacityType := "testCapacityType"
k8sVersion := "1.19"
diskSize := int64(100)
taintEffect1 := "effect 1"
taintKey1 := "key 1"
taintValue1 := "value 1"
taint1 := eks.Taint{
Effect: &taintEffect1,
Key: &taintKey1,
Value: &taintValue1,
}
taintEffect2 := "effect 2"
taintKey2 := "key 2"
taintValue2 := "value 2"
taint2 := eks.Taint{
Effect: &taintEffect2,
Key: &taintKey2,
Value: &taintValue2,
}
// Create test nodegroup
testNodegroup := eks.Nodegroup{
AmiType: &amiType,
ClusterName: &clusterName,
DiskSize: &diskSize,
Labels: nil,
NodegroupName: &nodegroupName,
CapacityType: &capacityType,
Version: &k8sVersion,
Taints: []*eks.Taint{&taint1, &taint2},
}
k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil)
c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k})
taintsList, err := c.getManagedNodegroupTaints(nodegroupName, clusterName)
require.NoError(t, err)
assert.Equal(t, len(taintsList), 2)
assert.Equal(t, taintsList[0].Effect, apiv1.TaintEffect(taintEffect1))
assert.Equal(t, taintsList[0].Key, taintKey1)
assert.Equal(t, taintsList[0].Value, taintValue1)
assert.Equal(t, taintsList[1].Effect, apiv1.TaintEffect(taintEffect2))
assert.Equal(t, taintsList[1].Key, taintKey2)
assert.Equal(t, taintsList[1].Value, taintValue2)
k.AssertCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{
ClusterName: &clusterName,
NodegroupName: &nodegroupName,
})
}