kubernetes
/
autoscaler
mirror of https://github.com/kubernetes/autoscaler.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

encapsulate autodiscovery utils in a separate file and remove autodiscovery for agentpool mode

This commit is contained in:
Marwan Ahmed 2020-08-23 15:22:22 -07:00
parent 5e68c1274a
commit ee176fb4bf
5 changed files with 162 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
cluster-autoscaler/cloudprovider/azure/azure_autodiscovery.go Normal file
View File

@ -0,0 +1,87 @@
package azure
import (
"fmt"
"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
"strings"
)
const (
autoDiscovererTypeLabel = "label"
)
// A labelAutoDiscoveryConfig specifies how to auto-discover Azure scale sets.
type labelAutoDiscoveryConfig struct {
// Key-values to match on.
Selector map[string]string
}
// ParseLabelAutoDiscoverySpecs returns any provided NodeGroupAutoDiscoverySpecs
// parsed into configuration appropriate for ASG autodiscovery.
func ParseLabelAutoDiscoverySpecs(o cloudprovider.NodeGroupDiscoveryOptions) ([]labelAutoDiscoveryConfig, error) {
cfgs := make([]labelAutoDiscoveryConfig, len(o.NodeGroupAutoDiscoverySpecs))
var err error
for i, spec := range o.NodeGroupAutoDiscoverySpecs {
cfgs[i], err = parseLabelAutoDiscoverySpec(spec)
if err != nil {
return nil, err
}
}
return cfgs, nil
}
// parseLabelAutoDiscoverySpec parses a single spec and returns the corresponding node group spec.
func parseLabelAutoDiscoverySpec(spec string) (labelAutoDiscoveryConfig, error) {
cfg := labelAutoDiscoveryConfig{
Selector: make(map[string]string),
}
tokens := strings.Split(spec, ":")
if len(tokens) != 2 {
return cfg, fmt.Errorf("spec \"%s\" should be discoverer:key=value,key=value", spec)
}
discoverer := tokens[0]
if discoverer != autoDiscovererTypeLabel {
return cfg, fmt.Errorf("unsupported discoverer specified: %s", discoverer)
}
for _, arg := range strings.Split(tokens[1], ",") {
kv := strings.Split(arg, "=")
if len(kv) != 2 {
return cfg, fmt.Errorf("invalid key=value pair %s", kv)
}
k, v := kv[0], kv[1]
if k == "" || v == "" {
return cfg, fmt.Errorf("empty value not allowed in key=value tag pairs")
}
cfg.Selector[k] = v
}
return cfg, nil
}
func matchDiscoveryConfig(labels map[string]*string, configs []labelAutoDiscoveryConfig) bool {
if len(configs) == 0 {
return false
}
for _, c := range configs {
if len(c.Selector) == 0 {
return false
}
for k, v := range c.Selector {
value, ok := labels[k]
if !ok {
return false
}
if len(v) > 0 {
if value == nil || *value != v {
return false
}
}
}
}
return true
}

61
cluster-autoscaler/cloudprovider/azure/azure_autodiscovery_test.go Normal file
View File

@ -0,0 +1,61 @@
package azure
import (
"github.com/stretchr/testify/assert"
"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
"testing"
)
func TestParseLabelAutoDiscoverySpecs(t *testing.T) {
testCases := []struct {
name string
specs []string
expected []labelAutoDiscoveryConfig
expectedErr bool
}{
{
name: "ValidSpec",
specs: []string{
"label:cluster-autoscaler-enabled=true,cluster-autoscaler-name=fake-cluster",
"label:test-tag=test-value,another-test-tag=another-test-value",
},
expected: []labelAutoDiscoveryConfig{
{Selector: map[string]string{"cluster-autoscaler-enabled": "true", "cluster-autoscaler-name": "fake-cluster"}},
{Selector: map[string]string{"test-tag": "test-value", "another-test-tag": "another-test-value"}},
},
},
{
name: "MissingAutoDiscoverLabel",
specs: []string{"test-tag=test-value,another-test-tag"},
expectedErr: true,
},
{
name: "InvalidAutoDiscoerLabel",
specs: []string{"invalid:test-tag=test-value,another-test-tag"},
expectedErr: true,
},
{
name: "MissingValue",
specs: []string{"label:test-tag="},
expectedErr: true,
},
{
name: "MissingKey",
specs: []string{"label:=test-val"},
expectedErr: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
ngdo := cloudprovider.NodeGroupDiscoveryOptions{NodeGroupAutoDiscoverySpecs: tc.specs}
actual, err := ParseLabelAutoDiscoverySpecs(ngdo)
if tc.expectedErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.True(t, assert.ObjectsAreEqualValues(tc.expected, actual), "expected %#v, but found: %#v", tc.expected, actual)
})
}
}

109
cluster-autoscaler/cloudprovider/azure/azure_manager.go
View File

@ -39,24 +39,8 @@ const (
scaleToZeroSupportedStandard = false
scaleToZeroSupportedVMSS = true
refreshInterval = 1 * time.Minute
vmssTagMin = "min"
vmssTagMax = "max"
autoDiscovererTypeLabel = "label"
labelAutoDiscovererKeyMinNodes = "min"
labelAutoDiscovererKeyMaxNodes = "max"
)
var validLabelAutoDiscovererKeys = strings.Join([]string{
labelAutoDiscovererKeyMinNodes,
labelAutoDiscovererKeyMaxNodes,
}, ", ")
// A labelAutoDiscoveryConfig specifies how to autodiscover Azure scale sets.
type labelAutoDiscoveryConfig struct {
// Key-values to match on.
Selector map[string]string
}
// AzureManager handles Azure communication and data caching.
type AzureManager struct {
@ -71,7 +55,6 @@ type AzureManager struct {
}
// CreateAzureManager creates Azure Manager object to work with Azure.
func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.NodeGroupDiscoveryOptions) (*AzureManager, error) {
cfg, err := BuildAzureConfig(configReader)
@ -109,7 +92,7 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node
}
manager.asgCache = cache
specs, err := parseLabelAutoDiscoverySpecs(discoveryOpts)
specs, err := ParseLabelAutoDiscoverySpecs(discoveryOpts)
if err != nil {
return nil, err
}
@ -271,21 +254,11 @@ func (m *AzureManager) getFilteredAutoscalingGroups(filter []labelAutoDiscoveryC
return nil, nil
}
switch m.config.VMType {
case vmTypeVMSS:
asgs, err = m.listScaleSets(filter)
case vmTypeStandard:
asgs, err = m.listAgentPools(filter)
case vmTypeAKS:
return nil, nil
default:
err = fmt.Errorf("vmType %q not supported", m.config.VMType)
}
if err != nil {
return nil, err
if m.config.VMType == vmTypeVMSS {
return m.listScaleSets(filter)
}
return asgs, nil
return nil, fmt.Errorf("vmType %q does not support autodiscovery", m.config.VMType)
}
// listScaleSets gets a list of scale sets and instanceIDs.
@ -364,77 +337,3 @@ func (m *AzureManager) listScaleSets(filter []labelAutoDiscoveryConfig) ([]cloud
return asgs, nil
}
// listAgentPools gets a list of agent pools and instanceIDs.
// Note: filter won't take effect for agent pools.
func (m *AzureManager) listAgentPools(filter []labelAutoDiscoveryConfig) (asgs []cloudprovider.NodeGroup, err error) {
ctx, cancel := getContextWithCancel()
defer cancel()
deploy, err := m.azClient.deploymentsClient.Get(ctx, m.config.ResourceGroup, m.config.Deployment)
if err != nil {
klog.Errorf("deploymentsClient.Get(%s, %s) failed: %v", m.config.ResourceGroup, m.config.Deployment, err)
return nil, err
}
parameters := deploy.Properties.Parameters.(map[string]interface{})
for k := range parameters {
if k == "masterVMSize" || !strings.HasSuffix(k, "VMSize") {
continue
}
poolName := strings.TrimRight(k, "VMSize")
spec := &dynamic.NodeGroupSpec{
Name: poolName,
MinSize: 1,
MaxSize: -1,
SupportScaleToZero: scaleToZeroSupportedStandard,
}
asg, _ := NewAgentPool(spec, m)
asgs = append(asgs, asg)
}
return asgs, nil
}
// ParseLabelAutoDiscoverySpecs returns any provided NodeGroupAutoDiscoverySpecs
// parsed into configuration appropriate for ASG autodiscovery.
func parseLabelAutoDiscoverySpecs(o cloudprovider.NodeGroupDiscoveryOptions) ([]labelAutoDiscoveryConfig, error) {
cfgs := make([]labelAutoDiscoveryConfig, len(o.NodeGroupAutoDiscoverySpecs))
var err error
for i, spec := range o.NodeGroupAutoDiscoverySpecs {
cfgs[i], err = parseLabelAutoDiscoverySpec(spec)
if err != nil {
return nil, err
}
}
return cfgs, nil
}
// parseLabelAutoDiscoverySpec parses a single spec and returns the corredponding node group spec.
func parseLabelAutoDiscoverySpec(spec string) (labelAutoDiscoveryConfig, error) {
cfg := labelAutoDiscoveryConfig{
Selector: make(map[string]string),
}
tokens := strings.Split(spec, ":")
if len(tokens) != 2 {
return cfg, fmt.Errorf("spec \"%s\" should be discoverer:key=value,key=value", spec)
}
discoverer := tokens[0]
if discoverer != autoDiscovererTypeLabel {
return cfg, fmt.Errorf("unsupported discoverer specified: %s", discoverer)
}
for _, arg := range strings.Split(tokens[1], ",") {
kv := strings.Split(arg, "=")
if len(kv) != 2 {
return cfg, fmt.Errorf("invalid key=value pair %s", kv)
}
k, v := kv[0], kv[1]
if k == "" || v == "" {
return cfg, fmt.Errorf("empty value not allowed in key=value tag pairs")
}
cfg.Selector[k] = v
}
return cfg, nil
}

120
cluster-autoscaler/cloudprovider/azure/azure_manager_test.go
View File

@ -304,54 +304,6 @@ func TestCreateAzureManagerValidConfigForStandardVMType(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, *expectedConfig, *manager.config, "unexpected azure manager configuration, expected: %v, actual: %v", *expectedConfig, *manager.config)
discoveryOpts := cloudprovider.NodeGroupDiscoveryOptions{NodeGroupAutoDiscoverySpecs: []string{
"label:cluster-autoscaler-enabled=true,cluster-autoscaler-name=fake-cluster",
"label:test-tag=test-value,another-test-tag=another-test-value",
}}
timeLayout := "2006-01-02 15:04:05"
timeBenchMark, _ := time.Parse(timeLayout, "2000-01-01 00:00:00")
fakeDeployments := map[string]resources.DeploymentExtended{
"cluster-autoscaler-0001": {
Name: to.StringPtr("cluster-autoscaler-0001"),
Properties: &resources.DeploymentPropertiesExtended{
ProvisioningState: to.StringPtr("Succeeded"),
Parameters: map[string]interface{}{
"PoolName01VMSize": to.StringPtr("PoolName01"),
},
Template: map[string]interface{}{
"resources": []interface{}{
map[string]interface{}{
"type": "Microsoft.Compute/virtualMachines/extensions",
"name": "cluster-autoscaler-0001-resourceName",
"properties": map[string]interface{}{
"hardwareProfile": map[string]interface{}{
"VMSize": "10G",
},
},
},
},
},
Timestamp: &date.Time{Time: timeBenchMark},
},
},
}
manager.azClient.deploymentsClient = &DeploymentsClientMock{
FakeStore: fakeDeployments,
}
specs, err2 := parseLabelAutoDiscoverySpecs(discoveryOpts)
assert.NoError(t, err2)
result, err3 := manager.getFilteredAutoscalingGroups(specs)
expectedNodeGroup := []cloudprovider.NodeGroup{(*AgentPool)(nil)}
assert.NoError(t, err3)
assert.Equal(t, expectedNodeGroup, result, "NodeGroup does not match, expected: %v, actual: %v", expectedNodeGroup, result)
// parseLabelAutoDiscoverySpecs with invalid NodeGroupDiscoveryOptions
invalidDiscoveryOpts := cloudprovider.NodeGroupDiscoveryOptions{NodeGroupAutoDiscoverySpecs: []string{"label:keywithoutvalue"}}
specs, err4 := parseLabelAutoDiscoverySpecs(invalidDiscoveryOpts)
expectedCfg := []labelAutoDiscoveryConfig([]labelAutoDiscoveryConfig(nil))
expectedErr := fmt.Errorf("invalid key=value pair [keywithoutvalue]")
assert.Equal(t, expectedCfg, specs, "Return labelAutoDiscoveryConfig does not match, expected: %v, actual: %v", expectedCfg, specs)
assert.Equal(t, expectedErr, err4, "parseLabelAutoDiscoverySpecs return error does not match, expected: %v, actual: %v", expectedErr, err4)
}
func TestCreateAzureManagerValidConfigForStandardVMTypeWithoutDeploymentParameters(t *testing.T) {
@ -679,59 +631,6 @@ func TestFetchExplicitAsgs(t *testing.T) {
assert.Equal(t, expectedErr, err, "manager.fetchExplicitAsgs return error does not match, expected: %v, actual: %v", expectedErr, err)
}
func TestParseLabelAutoDiscoverySpecs(t *testing.T) {
testCases := []struct {
name string
specs []string
expected []labelAutoDiscoveryConfig
expectedErr bool
}{
{
name: "ValidSpec",
specs: []string{
"label:cluster-autoscaler-enabled=true,cluster-autoscaler-name=fake-cluster",
"label:test-tag=test-value,another-test-tag=another-test-value",
},
expected: []labelAutoDiscoveryConfig{
{Selector: map[string]string{"cluster-autoscaler-enabled": "true", "cluster-autoscaler-name": "fake-cluster"}},
{Selector: map[string]string{"test-tag": "test-value", "another-test-tag": "another-test-value"}},
},
},
{
name: "MissingAutoDiscoverLabel",
specs: []string{"test-tag=test-value,another-test-tag"},
expectedErr: true,
},
{
name: "InvalidAutoDiscoerLabel",
specs: []string{"invalid:test-tag=test-value,another-test-tag"},
expectedErr: true,
},
{
name: "MissingValue",
specs: []string{"label:test-tag="},
expectedErr: true,
},
{
name: "MissingKey",
specs: []string{"label:=test-val"},
expectedErr: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
ngdo := cloudprovider.NodeGroupDiscoveryOptions{NodeGroupAutoDiscoverySpecs: tc.specs}
actual, err := parseLabelAutoDiscoverySpecs(ngdo)
if tc.expectedErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.True(t, assert.ObjectsAreEqualValues(tc.expected, actual), "expected %#v, but found: %#v", tc.expected, actual)
})
}
}
func TestListScalesets(t *testing.T) {
ctrl := gomock.NewController(t)
@ -745,7 +644,7 @@ func TestListScalesets(t *testing.T) {
ngdo := cloudprovider.NodeGroupDiscoveryOptions{
NodeGroupAutoDiscoverySpecs: []string{fmt.Sprintf("label:%s=%s", vmssTag, vmssTagValue)},
}
specs, err := parseLabelAutoDiscoverySpecs(ngdo)
specs, err := ParseLabelAutoDiscoverySpecs(ngdo)
assert.NoError(t, err)
testCases := []struct {
@ -860,7 +759,7 @@ func TestGetFilteredAutoscalingGroupsVmss(t *testing.T) {
mockVMSSClient.EXPECT().List(gomock.Any(), manager.config.ResourceGroup).Return(expectedScaleSets, nil).AnyTimes()
manager.azClient.virtualMachineScaleSetsClient = mockVMSSClient
specs, err := parseLabelAutoDiscoverySpecs(ngdo)
specs, err := ParseLabelAutoDiscoverySpecs(ngdo)
assert.NoError(t, err)
asgs, err := manager.getFilteredAutoscalingGroups(specs)
@ -894,16 +793,17 @@ func TestGetFilteredAutoscalingGroupsWithInvalidVMType(t *testing.T) {
manager.azClient.virtualMachineScaleSetsClient = mockVMSSClient
manager.config.VMType = vmTypeAKS
specs, err := parseLabelAutoDiscoverySpecs(ngdo)
specs, err := ParseLabelAutoDiscoverySpecs(ngdo)
assert.NoError(t, err)
asgs1, err1 := manager.getFilteredAutoscalingGroups(specs)
assert.Nil(t, asgs1)
assert.Nil(t, err1)
expectedErr := fmt.Errorf("vmType \"aks\" does not support autodiscovery")
asgs, err2 := manager.getFilteredAutoscalingGroups(specs)
assert.Nil(t, asgs)
assert.Equal(t, expectedErr, err2, "Not match, expected: %v, actual: %v", expectedErr, err2)
manager.config.VMType = "invalidVMType"
expectedErr := fmt.Errorf("vmType \"invalidVMType\" not supported")
asgs, err2 := manager.getFilteredAutoscalingGroups(specs)
expectedErr = fmt.Errorf("vmType \"invalidVMType\" does not support autodiscovery")
asgs, err2 = manager.getFilteredAutoscalingGroups(specs)
assert.Nil(t, asgs)
assert.Equal(t, expectedErr, err2, "Not match, expected: %v, actual: %v", expectedErr, err2)
}
@ -935,7 +835,7 @@ func TestFetchAutoAsgsVmss(t *testing.T) {
mockVMSSVMClient.EXPECT().List(gomock.Any(), manager.config.ResourceGroup, vmssName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
manager.azClient.virtualMachineScaleSetVMsClient = mockVMSSVMClient
specs, err := parseLabelAutoDiscoverySpecs(ngdo)
specs, err := ParseLabelAutoDiscoverySpecs(ngdo)
assert.NoError(t, err)
manager.asgAutoDiscoverySpecs = specs

27
cluster-autoscaler/cloudprovider/azure/azure_util.go
View File

@ -495,33 +495,6 @@ func GetVMNameIndex(osType compute.OperatingSystemTypes, vmName string) (int, er
return agentIndex, nil
}
func matchDiscoveryConfig(labels map[string]*string, configs []labelAutoDiscoveryConfig) bool {
if len(configs) == 0 {
return false
}
for _, c := range configs {
if len(c.Selector) == 0 {
return false
}
for k, v := range c.Selector {
value, ok := labels[k]
if !ok {
return false
}
if len(v) > 0 {
if value == nil || *value != v {
return false
}
}
}
}
return true
}
// getLastSegment gets the last segment (splitting by '/'.)
func getLastSegment(ID string) (string, error) {