Merge 16cfe50486 into a9292351c3
This commit is contained in:
commit
3aedfc9929
|
|
@ -204,17 +204,17 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch bool
|
shouldMatch bool
|
||||||
}{{
|
}{{
|
||||||
name: "no clustername, namespace, or label selector specified should match any MachineSet",
|
name: "no clustername, namespace, or label selector specified should match any MachineSet",
|
||||||
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil),
|
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{labelSelector: labels.NewSelector()},
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{labelSelector: labels.NewSelector()},
|
||||||
shouldMatch: true,
|
shouldMatch: true,
|
||||||
}, {
|
}, {
|
||||||
name: "no clustername, namespace, or label selector specified should match any MachineDeployment",
|
name: "no clustername, namespace, or label selector specified should match any MachineDeployment",
|
||||||
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil),
|
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{labelSelector: labels.NewSelector()},
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{labelSelector: labels.NewSelector()},
|
||||||
shouldMatch: true,
|
shouldMatch: true,
|
||||||
}, {
|
}, {
|
||||||
name: "clustername specified does not match MachineSet, namespace matches, no labels specified",
|
name: "clustername specified does not match MachineSet, namespace matches, no labels specified",
|
||||||
testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, false, nil, nil),
|
testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, false, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
namespace: "default",
|
namespace: "default",
|
||||||
|
|
@ -223,7 +223,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch: false,
|
shouldMatch: false,
|
||||||
}, {
|
}, {
|
||||||
name: "clustername specified does not match MachineDeployment, namespace matches, no labels specified",
|
name: "clustername specified does not match MachineDeployment, namespace matches, no labels specified",
|
||||||
testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, true, nil, nil),
|
testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, true, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
namespace: "default",
|
namespace: "default",
|
||||||
|
|
@ -232,7 +232,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch: false,
|
shouldMatch: false,
|
||||||
}, {
|
}, {
|
||||||
name: "namespace specified does not match MachineSet, clusterName matches, no labels specified",
|
name: "namespace specified does not match MachineSet, clusterName matches, no labels specified",
|
||||||
testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, false, nil, nil),
|
testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, false, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
namespace: "default",
|
namespace: "default",
|
||||||
|
|
@ -241,7 +241,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch: false,
|
shouldMatch: false,
|
||||||
}, {
|
}, {
|
||||||
name: "clustername specified does not match MachineDeployment, namespace matches, no labels specified",
|
name: "clustername specified does not match MachineDeployment, namespace matches, no labels specified",
|
||||||
testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, true, nil, nil),
|
testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, true, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
namespace: "default",
|
namespace: "default",
|
||||||
|
|
@ -250,7 +250,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch: false,
|
shouldMatch: false,
|
||||||
}, {
|
}, {
|
||||||
name: "namespace and clusterName matches MachineSet, no labels specified",
|
name: "namespace and clusterName matches MachineSet, no labels specified",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
namespace: "default",
|
namespace: "default",
|
||||||
|
|
@ -259,7 +259,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch: true,
|
shouldMatch: true,
|
||||||
}, {
|
}, {
|
||||||
name: "namespace and clusterName matches MachineDeployment, no labels specified",
|
name: "namespace and clusterName matches MachineDeployment, no labels specified",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
namespace: "default",
|
namespace: "default",
|
||||||
|
|
@ -268,7 +268,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch: true,
|
shouldMatch: true,
|
||||||
}, {
|
}, {
|
||||||
name: "namespace and clusterName matches MachineSet, does not match label selector",
|
name: "namespace and clusterName matches MachineSet, does not match label selector",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
namespace: "default",
|
namespace: "default",
|
||||||
|
|
@ -277,7 +277,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch: false,
|
shouldMatch: false,
|
||||||
}, {
|
}, {
|
||||||
name: "namespace and clusterName matches MachineDeployment, does not match label selector",
|
name: "namespace and clusterName matches MachineDeployment, does not match label selector",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
namespace: "default",
|
namespace: "default",
|
||||||
|
|
@ -286,7 +286,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch: false,
|
shouldMatch: false,
|
||||||
}, {
|
}, {
|
||||||
name: "namespace, clusterName, and label selector matches MachineSet",
|
name: "namespace, clusterName, and label selector matches MachineSet",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
|
||||||
additionalLabels: map[string]string{"color": "green"},
|
additionalLabels: map[string]string{"color": "green"},
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
|
|
@ -296,7 +296,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
|
||||||
shouldMatch: true,
|
shouldMatch: true,
|
||||||
}, {
|
}, {
|
||||||
name: "namespace, clusterName, and label selector matches MachineDeployment",
|
name: "namespace, clusterName, and label selector matches MachineDeployment",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
|
||||||
additionalLabels: map[string]string{"color": "green"},
|
additionalLabels: map[string]string{"color": "green"},
|
||||||
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
|
||||||
clusterName: "foo",
|
clusterName: "foo",
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,7 @@ type testSpec struct {
|
||||||
namespace string
|
namespace string
|
||||||
nodeCount int
|
nodeCount int
|
||||||
rootIsMachineDeployment bool
|
rootIsMachineDeployment bool
|
||||||
|
nodeInfo map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
const customCAPIGroup = "custom.x-k8s.io"
|
const customCAPIGroup = "custom.x-k8s.io"
|
||||||
|
|
@ -307,33 +308,33 @@ func mustCreateTestController(t testing.TB, testConfigs ...*testConfig) (*machin
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func createMachineSetTestConfig(namespace, clusterName, namePrefix string, nodeCount int, annotations map[string]string, capacity map[string]string) *testConfig {
|
func createMachineSetTestConfig(namespace, clusterName, namePrefix string, nodeCount int, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) *testConfig {
|
||||||
return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, 1, nodeCount, false, annotations, capacity)...)[0]
|
return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, 1, nodeCount, false, annotations, capacity, nodeInfo)...)[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
func createMachineSetTestConfigs(namespace, clusterName, namePrefix string, configCount, nodeCount int, annotations map[string]string, capacity map[string]string) []*testConfig {
|
func createMachineSetTestConfigs(namespace, clusterName, namePrefix string, configCount, nodeCount int, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) []*testConfig {
|
||||||
return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, configCount, nodeCount, false, annotations, capacity)...)
|
return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, configCount, nodeCount, false, annotations, capacity, nodeInfo)...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createMachineDeploymentTestConfig(namespace, clusterName, namePrefix string, nodeCount int, annotations map[string]string, capacity map[string]string) *testConfig {
|
func createMachineDeploymentTestConfig(namespace, clusterName, namePrefix string, nodeCount int, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) *testConfig {
|
||||||
return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, 1, nodeCount, true, annotations, capacity)...)[0]
|
return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, 1, nodeCount, true, annotations, capacity, nodeInfo)...)[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
func createMachineDeploymentTestConfigs(namespace, clusterName, namePrefix string, configCount, nodeCount int, annotations map[string]string, capacity map[string]string) []*testConfig {
|
func createMachineDeploymentTestConfigs(namespace, clusterName, namePrefix string, configCount, nodeCount int, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) []*testConfig {
|
||||||
return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, configCount, nodeCount, true, annotations, capacity)...)
|
return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, configCount, nodeCount, true, annotations, capacity, nodeInfo)...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createTestSpecs(namespace, clusterName, namePrefix string, scalableResourceCount, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string) []testSpec {
|
func createTestSpecs(namespace, clusterName, namePrefix string, scalableResourceCount, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) []testSpec {
|
||||||
var specs []testSpec
|
var specs []testSpec
|
||||||
|
|
||||||
for i := 0; i < scalableResourceCount; i++ {
|
for i := 0; i < scalableResourceCount; i++ {
|
||||||
specs = append(specs, createTestSpec(namespace, clusterName, fmt.Sprintf("%s-%d", namePrefix, i), nodeCount, isMachineDeployment, annotations, capacity))
|
specs = append(specs, createTestSpec(namespace, clusterName, fmt.Sprintf("%s-%d", namePrefix, i), nodeCount, isMachineDeployment, annotations, capacity, nodeInfo))
|
||||||
}
|
}
|
||||||
|
|
||||||
return specs
|
return specs
|
||||||
}
|
}
|
||||||
|
|
||||||
func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string) testSpec {
|
func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) testSpec {
|
||||||
return testSpec{
|
return testSpec{
|
||||||
annotations: annotations,
|
annotations: annotations,
|
||||||
capacity: capacity,
|
capacity: capacity,
|
||||||
|
|
@ -343,6 +344,7 @@ func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachin
|
||||||
namespace: namespace,
|
namespace: namespace,
|
||||||
nodeCount: nodeCount,
|
nodeCount: nodeCount,
|
||||||
rootIsMachineDeployment: isMachineDeployment,
|
rootIsMachineDeployment: isMachineDeployment,
|
||||||
|
nodeInfo: nodeInfo,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -451,9 +453,7 @@ func createTestConfigs(specs ...testSpec) []*testConfig {
|
||||||
Kind: config.machineSet.GetKind(),
|
Kind: config.machineSet.GetKind(),
|
||||||
UID: config.machineSet.GetUID(),
|
UID: config.machineSet.GetUID(),
|
||||||
}
|
}
|
||||||
|
if spec.capacity != nil || spec.nodeInfo != nil {
|
||||||
if spec.capacity != nil {
|
|
||||||
klog.V(4).Infof("adding capacity to machine template")
|
|
||||||
config.machineTemplate = &unstructured.Unstructured{
|
config.machineTemplate = &unstructured.Unstructured{
|
||||||
Object: map[string]interface{}{
|
Object: map[string]interface{}{
|
||||||
"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
|
"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
|
||||||
|
|
@ -465,6 +465,10 @@ func createTestConfigs(specs ...testSpec) []*testConfig {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
if spec.capacity != nil {
|
||||||
|
klog.V(4).Infof("adding capacity to machine template")
|
||||||
|
|
||||||
if err := unstructured.SetNestedStringMap(config.machineTemplate.Object, spec.capacity, "status", "capacity"); err != nil {
|
if err := unstructured.SetNestedStringMap(config.machineTemplate.Object, spec.capacity, "status", "capacity"); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
@ -472,6 +476,16 @@ func createTestConfigs(specs ...testSpec) []*testConfig {
|
||||||
klog.V(4).Infof("not adding capacity")
|
klog.V(4).Infof("not adding capacity")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if spec.nodeInfo != nil {
|
||||||
|
klog.V(4).Infof("adding nodeInfo to machine template")
|
||||||
|
|
||||||
|
if err := unstructured.SetNestedStringMap(config.machineTemplate.Object, spec.nodeInfo, "status", "nodeInfo"); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
klog.V(4).Infof("not adding nodeInfo")
|
||||||
|
}
|
||||||
|
|
||||||
for j := 0; j < spec.nodeCount; j++ {
|
for j := 0; j < spec.nodeCount; j++ {
|
||||||
config.nodes[j], config.machines[j] = makeLinkedNodeAndMachine(j, spec.namespace, spec.clusterName, machineOwner, machineSetLabels)
|
config.nodes[j], config.machines[j] = makeLinkedNodeAndMachine(j, spec.namespace, spec.clusterName, machineOwner, machineSetLabels)
|
||||||
}
|
}
|
||||||
|
|
@ -694,7 +708,7 @@ func TestControllerFindMachine(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
if tc.name == "" {
|
if tc.name == "" {
|
||||||
tc.name = testConfig.machines[0].GetName()
|
tc.name = testConfig.machines[0].GetName()
|
||||||
}
|
}
|
||||||
|
|
@ -722,7 +736,7 @@ func TestControllerFindMachineOwner(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
controller, stop := mustCreateTestController(t, testConfig)
|
controller, stop := mustCreateTestController(t, testConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
@ -771,7 +785,7 @@ func TestControllerFindMachineByProviderID(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
controller, stop := mustCreateTestController(t, testConfig)
|
controller, stop := mustCreateTestController(t, testConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
@ -833,7 +847,7 @@ func TestControllerFindNodeByNodeName(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
controller, stop := mustCreateTestController(t, testConfig)
|
controller, stop := mustCreateTestController(t, testConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
@ -933,7 +947,7 @@ func TestControllerListMachinesForScalableResource(t *testing.T) {
|
||||||
testConfig1 := createMachineSetTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{
|
testConfig1 := createMachineSetTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
// Construct a second set of objects and add the machines,
|
// Construct a second set of objects and add the machines,
|
||||||
// nodes and the additional machineset to the existing set of
|
// nodes and the additional machineset to the existing set of
|
||||||
|
|
@ -942,7 +956,7 @@ func TestControllerListMachinesForScalableResource(t *testing.T) {
|
||||||
testConfig2 := createMachineSetTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{
|
testConfig2 := createMachineSetTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
test(t, testConfig1, testConfig2)
|
test(t, testConfig1, testConfig2)
|
||||||
})
|
})
|
||||||
|
|
@ -953,7 +967,7 @@ func TestControllerListMachinesForScalableResource(t *testing.T) {
|
||||||
testConfig1 := createMachineDeploymentTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{
|
testConfig1 := createMachineDeploymentTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
// Construct a second set of objects and add the machines,
|
// Construct a second set of objects and add the machines,
|
||||||
// nodes, machineset, and the additional machineset to the existing set of
|
// nodes, machineset, and the additional machineset to the existing set of
|
||||||
|
|
@ -962,7 +976,7 @@ func TestControllerListMachinesForScalableResource(t *testing.T) {
|
||||||
testConfig2 := createMachineDeploymentTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{
|
testConfig2 := createMachineDeploymentTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
test(t, testConfig1, testConfig2)
|
test(t, testConfig1, testConfig2)
|
||||||
})
|
})
|
||||||
|
|
@ -993,7 +1007,7 @@ func TestControllerLookupNodeGroupForNonExistentNode(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
test(t, testConfig)
|
test(t, testConfig)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -1001,7 +1015,7 @@ func TestControllerLookupNodeGroupForNonExistentNode(t *testing.T) {
|
||||||
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
test(t, testConfig)
|
test(t, testConfig)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -1032,7 +1046,7 @@ func TestControllerNodeGroupForNodeWithMissingMachineOwner(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
test(t, testConfig)
|
test(t, testConfig)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -1040,7 +1054,7 @@ func TestControllerNodeGroupForNodeWithMissingMachineOwner(t *testing.T) {
|
||||||
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
test(t, testConfig)
|
test(t, testConfig)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -1049,7 +1063,7 @@ func TestControllerNodeGroupForNodeWithMissingSetMachineOwner(t *testing.T) {
|
||||||
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
controller, stop := mustCreateTestController(t, testConfig)
|
controller, stop := mustCreateTestController(t, testConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
@ -1090,7 +1104,7 @@ func TestControllerNodeGroupForNodeWithPositiveScalingBounds(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "1",
|
nodeGroupMaxSizeAnnotationKey: "1",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
test(t, testConfig)
|
test(t, testConfig)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -1098,7 +1112,7 @@ func TestControllerNodeGroupForNodeWithPositiveScalingBounds(t *testing.T) {
|
||||||
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "1",
|
nodeGroupMaxSizeAnnotationKey: "1",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
test(t, testConfig)
|
test(t, testConfig)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -1130,14 +1144,14 @@ func TestControllerNodeGroups(t *testing.T) {
|
||||||
assertNodegroupLen(t, controller, 0)
|
assertNodegroupLen(t, controller, 0)
|
||||||
|
|
||||||
// Test #2: add 5 machineset-based nodegroups
|
// Test #2: add 5 machineset-based nodegroups
|
||||||
machineSetConfigs := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil)
|
machineSetConfigs := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil, nil)
|
||||||
if err := addTestConfigs(t, controller, machineSetConfigs...); err != nil {
|
if err := addTestConfigs(t, controller, machineSetConfigs...); err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
assertNodegroupLen(t, controller, 5)
|
assertNodegroupLen(t, controller, 5)
|
||||||
|
|
||||||
// Test #2: add 2 machinedeployment-based nodegroups
|
// Test #2: add 2 machinedeployment-based nodegroups
|
||||||
machineDeploymentConfigs := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil)
|
machineDeploymentConfigs := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil, nil)
|
||||||
if err := addTestConfigs(t, controller, machineDeploymentConfigs...); err != nil {
|
if err := addTestConfigs(t, controller, machineDeploymentConfigs...); err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -1161,14 +1175,14 @@ func TestControllerNodeGroups(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test #5: 5 machineset with minSize=maxSize results in a five nodegroup
|
// Test #5: 5 machineset with minSize=maxSize results in a five nodegroup
|
||||||
machineSetConfigs = createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil)
|
machineSetConfigs = createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil, nil)
|
||||||
if err := addTestConfigs(t, controller, machineSetConfigs...); err != nil {
|
if err := addTestConfigs(t, controller, machineSetConfigs...); err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
assertNodegroupLen(t, controller, 5)
|
assertNodegroupLen(t, controller, 5)
|
||||||
|
|
||||||
// Test #6: add 2 machinedeployment with minSize=maxSize
|
// Test #6: add 2 machinedeployment with minSize=maxSize
|
||||||
machineDeploymentConfigs = createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil)
|
machineDeploymentConfigs = createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil, nil)
|
||||||
if err := addTestConfigs(t, controller, machineDeploymentConfigs...); err != nil {
|
if err := addTestConfigs(t, controller, machineDeploymentConfigs...); err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -1180,7 +1194,7 @@ func TestControllerNodeGroups(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test #7: machineset with bad scaling bounds results in an error and no nodegroups
|
// Test #7: machineset with bad scaling bounds results in an error and no nodegroups
|
||||||
machineSetConfigs = createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil)
|
machineSetConfigs = createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil, nil)
|
||||||
if err := addTestConfigs(t, controller, machineSetConfigs...); err != nil {
|
if err := addTestConfigs(t, controller, machineSetConfigs...); err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -1189,7 +1203,7 @@ func TestControllerNodeGroups(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test #8: machinedeployment with bad scaling bounds results in an error and no nodegroups
|
// Test #8: machinedeployment with bad scaling bounds results in an error and no nodegroups
|
||||||
machineDeploymentConfigs = createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil)
|
machineDeploymentConfigs = createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil, nil)
|
||||||
if err := addTestConfigs(t, controller, machineDeploymentConfigs...); err != nil {
|
if err := addTestConfigs(t, controller, machineDeploymentConfigs...); err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -1253,13 +1267,13 @@ func TestControllerNodeGroupsNodeCount(t *testing.T) {
|
||||||
|
|
||||||
t.Run("MachineSet", func(t *testing.T) {
|
t.Run("MachineSet", func(t *testing.T) {
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
test(t, tc, createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), tc.nodeGroups, tc.nodesPerGroup, annotations, nil))
|
test(t, tc, createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), tc.nodeGroups, tc.nodesPerGroup, annotations, nil, nil))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("MachineDeployment", func(t *testing.T) {
|
t.Run("MachineDeployment", func(t *testing.T) {
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
test(t, tc, createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), tc.nodeGroups, tc.nodesPerGroup, annotations, nil))
|
test(t, tc, createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), tc.nodeGroups, tc.nodesPerGroup, annotations, nil, nil))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -1268,7 +1282,7 @@ func TestControllerFindMachineFromNodeAnnotation(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
controller, stop := mustCreateTestController(t, testConfig)
|
controller, stop := mustCreateTestController(t, testConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
@ -1316,7 +1330,7 @@ func TestControllerMachineSetNodeNamesWithoutLinkage(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 3, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 3, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
controller, stop := mustCreateTestController(t, testConfig)
|
controller, stop := mustCreateTestController(t, testConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
@ -1365,7 +1379,7 @@ func TestControllerMachineSetNodeNamesUsingProviderID(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 3, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 3, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
controller, stop := mustCreateTestController(t, testConfig)
|
controller, stop := mustCreateTestController(t, testConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
@ -1404,7 +1418,7 @@ func TestControllerMachineSetNodeNamesUsingStatusNodeRefName(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 3, map[string]string{
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 3, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
controller, stop := mustCreateTestController(t, testConfig)
|
controller, stop := mustCreateTestController(t, testConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
@ -1494,7 +1508,7 @@ func TestControllerGetAPIVersionGroupWithMachineDeployments(t *testing.T) {
|
||||||
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "1",
|
nodeGroupMaxSizeAnnotationKey: "1",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
t.Setenv(CAPIGroupEnvVar, customCAPIGroup)
|
t.Setenv(CAPIGroupEnvVar, customCAPIGroup)
|
||||||
|
|
||||||
testConfig.machineDeployment.SetAPIVersion(fmt.Sprintf("%s/v1beta1", customCAPIGroup))
|
testConfig.machineDeployment.SetAPIVersion(fmt.Sprintf("%s/v1beta1", customCAPIGroup))
|
||||||
|
|
@ -1756,7 +1770,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
|
||||||
shouldMatch bool
|
shouldMatch bool
|
||||||
}{{
|
}{{
|
||||||
name: "autodiscovery specs includes permissive spec that should match any MachineSet",
|
name: "autodiscovery specs includes permissive spec that should match any MachineSet",
|
||||||
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil),
|
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil, nil),
|
||||||
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
||||||
{labelSelector: labels.NewSelector()},
|
{labelSelector: labels.NewSelector()},
|
||||||
{clusterName: "foo", namespace: "bar", labelSelector: labels.Nothing()},
|
{clusterName: "foo", namespace: "bar", labelSelector: labels.Nothing()},
|
||||||
|
|
@ -1764,7 +1778,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
|
||||||
shouldMatch: true,
|
shouldMatch: true,
|
||||||
}, {
|
}, {
|
||||||
name: "autodiscovery specs includes permissive spec that should match any MachineDeployment",
|
name: "autodiscovery specs includes permissive spec that should match any MachineDeployment",
|
||||||
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil),
|
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil, nil),
|
||||||
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
||||||
{labelSelector: labels.NewSelector()},
|
{labelSelector: labels.NewSelector()},
|
||||||
{clusterName: "foo", namespace: "bar", labelSelector: labels.Nothing()},
|
{clusterName: "foo", namespace: "bar", labelSelector: labels.Nothing()},
|
||||||
|
|
@ -1772,7 +1786,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
|
||||||
shouldMatch: true,
|
shouldMatch: true,
|
||||||
}, {
|
}, {
|
||||||
name: "autodiscovery specs includes a restrictive spec that should match specific MachineSet",
|
name: "autodiscovery specs includes a restrictive spec that should match specific MachineSet",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
|
||||||
additionalLabels: map[string]string{"color": "green"},
|
additionalLabels: map[string]string{"color": "green"},
|
||||||
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
||||||
{clusterName: "foo", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "green"})},
|
{clusterName: "foo", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "green"})},
|
||||||
|
|
@ -1781,7 +1795,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
|
||||||
shouldMatch: true,
|
shouldMatch: true,
|
||||||
}, {
|
}, {
|
||||||
name: "autodiscovery specs includes a restrictive spec that should match specific MachineDeployment",
|
name: "autodiscovery specs includes a restrictive spec that should match specific MachineDeployment",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
|
||||||
additionalLabels: map[string]string{"color": "green"},
|
additionalLabels: map[string]string{"color": "green"},
|
||||||
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
||||||
{clusterName: "foo", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "green"})},
|
{clusterName: "foo", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "green"})},
|
||||||
|
|
@ -1790,7 +1804,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
|
||||||
shouldMatch: true,
|
shouldMatch: true,
|
||||||
}, {
|
}, {
|
||||||
name: "autodiscovery specs does not include any specs that should match specific MachineSet",
|
name: "autodiscovery specs does not include any specs that should match specific MachineSet",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
|
||||||
additionalLabels: map[string]string{"color": "green"},
|
additionalLabels: map[string]string{"color": "green"},
|
||||||
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
||||||
{clusterName: "test", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "blue"})},
|
{clusterName: "test", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "blue"})},
|
||||||
|
|
@ -1799,7 +1813,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
|
||||||
shouldMatch: false,
|
shouldMatch: false,
|
||||||
}, {
|
}, {
|
||||||
name: "autodiscovery specs does not include any specs that should match specific MachineDeployment",
|
name: "autodiscovery specs does not include any specs that should match specific MachineDeployment",
|
||||||
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
|
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
|
||||||
additionalLabels: map[string]string{"color": "green"},
|
additionalLabels: map[string]string{"color": "green"},
|
||||||
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
|
||||||
{clusterName: "test", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "blue"})},
|
{clusterName: "test", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "blue"})},
|
||||||
|
|
@ -1829,9 +1843,9 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func Test_machineController_listScalableResources(t *testing.T) {
|
func Test_machineController_listScalableResources(t *testing.T) {
|
||||||
uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, nil)
|
uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, nil, nil)
|
||||||
|
|
||||||
mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, nil, nil)
|
mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, nil, nil, nil)
|
||||||
mdTestConfigs = append(mdTestConfigs, uniqueMDConfig)
|
mdTestConfigs = append(mdTestConfigs, uniqueMDConfig)
|
||||||
|
|
||||||
allMachineDeployments := make([]*unstructured.Unstructured, 0, len(mdTestConfigs))
|
allMachineDeployments := make([]*unstructured.Unstructured, 0, len(mdTestConfigs))
|
||||||
|
|
@ -1839,9 +1853,9 @@ func Test_machineController_listScalableResources(t *testing.T) {
|
||||||
allMachineDeployments = append(allMachineDeployments, mdTestConfigs[i].machineDeployment)
|
allMachineDeployments = append(allMachineDeployments, mdTestConfigs[i].machineDeployment)
|
||||||
}
|
}
|
||||||
|
|
||||||
uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, nil)
|
uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, nil, nil)
|
||||||
|
|
||||||
msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, nil, nil)
|
msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, nil, nil, nil)
|
||||||
msTestConfigs = append(msTestConfigs, uniqueMSConfig)
|
msTestConfigs = append(msTestConfigs, uniqueMSConfig)
|
||||||
|
|
||||||
allMachineSets := make([]*unstructured.Unstructured, 0, len(msTestConfigs))
|
allMachineSets := make([]*unstructured.Unstructured, 0, len(msTestConfigs))
|
||||||
|
|
@ -1944,23 +1958,23 @@ func Test_machineController_nodeGroupForNode(t *testing.T) {
|
||||||
uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{
|
mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
mdTestConfigs = append(mdTestConfigs, uniqueMDConfig)
|
mdTestConfigs = append(mdTestConfigs, uniqueMDConfig)
|
||||||
|
|
||||||
uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{
|
msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
msTestConfigs = append(msTestConfigs, uniqueMSConfig)
|
msTestConfigs = append(msTestConfigs, uniqueMSConfig)
|
||||||
|
|
||||||
allTestConfigs := append(mdTestConfigs, msTestConfigs...)
|
allTestConfigs := append(mdTestConfigs, msTestConfigs...)
|
||||||
|
|
@ -2040,12 +2054,12 @@ func Test_machineController_nodeGroups(t *testing.T) {
|
||||||
uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{
|
mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
mdTestConfigs = append(mdTestConfigs, uniqueMDConfig)
|
mdTestConfigs = append(mdTestConfigs, uniqueMDConfig)
|
||||||
|
|
||||||
allMachineDeployments := make([]*unstructured.Unstructured, 0, len(mdTestConfigs))
|
allMachineDeployments := make([]*unstructured.Unstructured, 0, len(mdTestConfigs))
|
||||||
|
|
@ -2056,12 +2070,12 @@ func Test_machineController_nodeGroups(t *testing.T) {
|
||||||
uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
|
|
||||||
msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{
|
msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}, nil)
|
}, nil, nil)
|
||||||
msTestConfigs = append(msTestConfigs, uniqueMSConfig)
|
msTestConfigs = append(msTestConfigs, uniqueMSConfig)
|
||||||
|
|
||||||
allMachineSets := make([]*unstructured.Unstructured, 0, len(msTestConfigs))
|
allMachineSets := make([]*unstructured.Unstructured, 0, len(msTestConfigs))
|
||||||
|
|
|
||||||
|
|
@ -361,12 +361,17 @@ func (ng *nodegroup) TemplateNodeInfo() (*framework.NodeInfo, error) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
nsi := ng.scalableResource.InstanceSystemInfo()
|
||||||
|
if nsi != nil {
|
||||||
|
node.Status.NodeInfo = *nsi
|
||||||
|
}
|
||||||
|
|
||||||
node.Status.Capacity = capacity
|
node.Status.Capacity = capacity
|
||||||
node.Status.Allocatable = capacity
|
node.Status.Allocatable = capacity
|
||||||
node.Status.Conditions = cloudprovider.BuildReadyConditions()
|
node.Status.Conditions = cloudprovider.BuildReadyConditions()
|
||||||
node.Spec.Taints = ng.scalableResource.Taints()
|
node.Spec.Taints = ng.scalableResource.Taints()
|
||||||
|
|
||||||
node.Labels, err = ng.buildTemplateLabels(nodeName)
|
node.Labels, err = ng.buildTemplateLabels(nodeName, nsi)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -380,8 +385,19 @@ func (ng *nodegroup) TemplateNodeInfo() (*framework.NodeInfo, error) {
|
||||||
return nodeInfo, nil
|
return nodeInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ng *nodegroup) buildTemplateLabels(nodeName string) (map[string]string, error) {
|
func (ng *nodegroup) buildTemplateLabels(nodeName string, nsi *corev1.NodeSystemInfo) (map[string]string, error) {
|
||||||
labels := cloudprovider.JoinStringMaps(buildGenericLabels(nodeName), ng.scalableResource.Labels())
|
nsiLabels := make(map[string]string)
|
||||||
|
if nsi != nil {
|
||||||
|
nsiLabels[corev1.LabelArchStable] = nsi.Architecture
|
||||||
|
nsiLabels[corev1.LabelOSStable] = nsi.OperatingSystem
|
||||||
|
}
|
||||||
|
|
||||||
|
// The order of priority is:
|
||||||
|
// - Labels set in existing nodes for not-autoscale-from-zero cases
|
||||||
|
// - Labels set in the labels capacity annotation of machine template, machine set, and machine deployment.
|
||||||
|
// - Values in the status.nodeSystemInfo of MachineTemplates
|
||||||
|
// - Generic/default labels set in the environment of the cluster autoscaler
|
||||||
|
labels := cloudprovider.JoinStringMaps(buildGenericLabels(nodeName), nsiLabels, ng.scalableResource.Labels())
|
||||||
|
|
||||||
nodes, err := ng.Nodes()
|
nodes, err := ng.Nodes()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
|
|
@ -214,7 +214,7 @@ func TestNodeGroupNewNodeGroupConstructor(t *testing.T) {
|
||||||
t.Run("MachineSet", func(t *testing.T) {
|
t.Run("MachineSet", func(t *testing.T) {
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
t.Run(tc.description, func(t *testing.T) {
|
t.Run(tc.description, func(t *testing.T) {
|
||||||
test(t, tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), tc.nodeCount, tc.annotations, nil))
|
test(t, tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), tc.nodeCount, tc.annotations, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -222,7 +222,7 @@ func TestNodeGroupNewNodeGroupConstructor(t *testing.T) {
|
||||||
t.Run("MachineDeployment", func(t *testing.T) {
|
t.Run("MachineDeployment", func(t *testing.T) {
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
t.Run(tc.description, func(t *testing.T) {
|
t.Run(tc.description, func(t *testing.T) {
|
||||||
test(t, tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), tc.nodeCount, tc.annotations, nil))
|
test(t, tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), tc.nodeCount, tc.annotations, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -309,7 +309,7 @@ func TestNodeGroupIncreaseSizeErrors(t *testing.T) {
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}
|
}
|
||||||
test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil))
|
test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -321,7 +321,7 @@ func TestNodeGroupIncreaseSizeErrors(t *testing.T) {
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}
|
}
|
||||||
test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil))
|
test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -390,7 +390,7 @@ func TestNodeGroupIncreaseSize(t *testing.T) {
|
||||||
expected: 4,
|
expected: 4,
|
||||||
delta: 1,
|
delta: 1,
|
||||||
}
|
}
|
||||||
test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil))
|
test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil))
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("MachineDeployment", func(t *testing.T) {
|
t.Run("MachineDeployment", func(t *testing.T) {
|
||||||
|
|
@ -400,7 +400,7 @@ func TestNodeGroupIncreaseSize(t *testing.T) {
|
||||||
expected: 4,
|
expected: 4,
|
||||||
delta: 1,
|
delta: 1,
|
||||||
}
|
}
|
||||||
test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil))
|
test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -689,13 +689,13 @@ func TestNodeGroupDecreaseTargetSize(t *testing.T) {
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
t.Run(tc.description, func(t *testing.T) {
|
t.Run(tc.description, func(t *testing.T) {
|
||||||
test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil))
|
test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
t.Run(tc.description, func(t *testing.T) {
|
t.Run(tc.description, func(t *testing.T) {
|
||||||
test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil))
|
test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -781,7 +781,7 @@ func TestNodeGroupDecreaseSizeErrors(t *testing.T) {
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}
|
}
|
||||||
test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil))
|
test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -793,7 +793,7 @@ func TestNodeGroupDecreaseSizeErrors(t *testing.T) {
|
||||||
nodeGroupMinSizeAnnotationKey: "1",
|
nodeGroupMinSizeAnnotationKey: "1",
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
}
|
}
|
||||||
test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil))
|
test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -883,6 +883,7 @@ func TestNodeGroupDeleteNodes(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
@ -900,6 +901,7 @@ func TestNodeGroupDeleteNodes(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
@ -972,16 +974,16 @@ func TestNodeGroupMachineSetDeleteNodesWithMismatchedNodes(t *testing.T) {
|
||||||
t.Run("MachineSet", func(t *testing.T) {
|
t.Run("MachineSet", func(t *testing.T) {
|
||||||
namespace := RandomString(6)
|
namespace := RandomString(6)
|
||||||
clusterName := RandomString(6)
|
clusterName := RandomString(6)
|
||||||
testConfig0 := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil)
|
testConfig0 := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil, nil)
|
||||||
testConfig1 := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil)
|
testConfig1 := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil, nil)
|
||||||
test(t, 2, append(testConfig0, testConfig1...))
|
test(t, 2, append(testConfig0, testConfig1...))
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("MachineDeployment", func(t *testing.T) {
|
t.Run("MachineDeployment", func(t *testing.T) {
|
||||||
namespace := RandomString(6)
|
namespace := RandomString(6)
|
||||||
clusterName := RandomString(6)
|
clusterName := RandomString(6)
|
||||||
testConfig0 := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil)
|
testConfig0 := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil, nil)
|
||||||
testConfig1 := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil)
|
testConfig1 := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil, nil)
|
||||||
test(t, 2, append(testConfig0, testConfig1...))
|
test(t, 2, append(testConfig0, testConfig1...))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -1163,6 +1165,7 @@ func TestNodeGroupDeleteNodesTwice(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
@ -1180,6 +1183,7 @@ func TestNodeGroupDeleteNodesTwice(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
@ -1315,6 +1319,7 @@ func TestNodeGroupDeleteNodesSequential(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
@ -1332,6 +1337,7 @@ func TestNodeGroupDeleteNodesSequential(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
@ -1418,6 +1424,7 @@ func TestNodeGroupWithFailedMachine(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
@ -1435,6 +1442,7 @@ func TestNodeGroupWithFailedMachine(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
@ -1667,6 +1675,7 @@ func TestNodeGroupTemplateNodeInfo(t *testing.T) {
|
||||||
10,
|
10,
|
||||||
cloudprovider.JoinStringMaps(enableScaleAnnotations, tc.nodeGroupAnnotations),
|
cloudprovider.JoinStringMaps(enableScaleAnnotations, tc.nodeGroupAnnotations),
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
tc.config,
|
tc.config,
|
||||||
)
|
)
|
||||||
|
|
@ -1682,6 +1691,7 @@ func TestNodeGroupTemplateNodeInfo(t *testing.T) {
|
||||||
10,
|
10,
|
||||||
cloudprovider.JoinStringMaps(enableScaleAnnotations, tc.nodeGroupAnnotations),
|
cloudprovider.JoinStringMaps(enableScaleAnnotations, tc.nodeGroupAnnotations),
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
tc.config,
|
tc.config,
|
||||||
)
|
)
|
||||||
|
|
@ -1792,6 +1802,7 @@ func TestNodeGroupGetOptions(t *testing.T) {
|
||||||
10,
|
10,
|
||||||
cloudprovider.JoinStringMaps(enableScaleAnnotations, annotations),
|
cloudprovider.JoinStringMaps(enableScaleAnnotations, annotations),
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
c.expected,
|
c.expected,
|
||||||
)
|
)
|
||||||
|
|
@ -1807,6 +1818,7 @@ func TestNodeGroupGetOptions(t *testing.T) {
|
||||||
10,
|
10,
|
||||||
cloudprovider.JoinStringMaps(enableScaleAnnotations, annotations),
|
cloudprovider.JoinStringMaps(enableScaleAnnotations, annotations),
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
c.expected,
|
c.expected,
|
||||||
)
|
)
|
||||||
|
|
@ -2009,6 +2021,7 @@ func TestNodeGroupNodesInstancesStatus(t *testing.T) {
|
||||||
tc.nodeCount,
|
tc.nodeCount,
|
||||||
annotations,
|
annotations,
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
@ -2028,6 +2041,7 @@ func TestNodeGroupNodesInstancesStatus(t *testing.T) {
|
||||||
tc.nodeCount,
|
tc.nodeCount,
|
||||||
annotations,
|
annotations,
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -110,7 +110,7 @@ func BenchmarkNodeGroups(b *testing.B) {
|
||||||
|
|
||||||
controller, stop := mustCreateTestController(b)
|
controller, stop := mustCreateTestController(b)
|
||||||
defer stop()
|
defer stop()
|
||||||
machineSetConfigs := createMachineSetTestConfigs("namespace", "", RandomString(6), 100, 1, annotations, nil)
|
machineSetConfigs := createMachineSetTestConfigs("namespace", "", RandomString(6), 100, 1, annotations, nil, nil)
|
||||||
if err := addTestConfigs(b, controller, machineSetConfigs...); err != nil {
|
if err := addTestConfigs(b, controller, machineSetConfigs...); err != nil {
|
||||||
b.Fatalf("unexpected error: %v", err)
|
b.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ import (
|
||||||
"path"
|
"path"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
@ -42,6 +43,8 @@ import (
|
||||||
type unstructuredScalableResource struct {
|
type unstructuredScalableResource struct {
|
||||||
controller *machineController
|
controller *machineController
|
||||||
unstructured *unstructured.Unstructured
|
unstructured *unstructured.Unstructured
|
||||||
|
infraObj *unstructured.Unstructured
|
||||||
|
infraMutex sync.RWMutex
|
||||||
maxSize int
|
maxSize int
|
||||||
minSize int
|
minSize int
|
||||||
autoscalingOptions map[string]string
|
autoscalingOptions map[string]string
|
||||||
|
|
@ -321,6 +324,17 @@ func (r unstructuredScalableResource) InstanceCapacity() (map[corev1.ResourceNam
|
||||||
return capacity, nil
|
return capacity, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InstanceSystemInfo sets the nodeSystemInfo from the infrastructure reference resource.
|
||||||
|
// If the infrastructure reference resource is not found, returns nil.
|
||||||
|
func (r unstructuredScalableResource) InstanceSystemInfo() *apiv1.NodeSystemInfo {
|
||||||
|
infraObj, err := r.readInfrastructureReferenceResource()
|
||||||
|
if err != nil || infraObj == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
nsiObj := systemInfoFromInfrastructureObject(infraObj)
|
||||||
|
return &nsiObj
|
||||||
|
}
|
||||||
|
|
||||||
func (r unstructuredScalableResource) InstanceResourceSlices(nodeName string) ([]*resourceapi.ResourceSlice, error) {
|
func (r unstructuredScalableResource) InstanceResourceSlices(nodeName string) ([]*resourceapi.ResourceSlice, error) {
|
||||||
var result []*resourceapi.ResourceSlice
|
var result []*resourceapi.ResourceSlice
|
||||||
driver := r.InstanceDRADriver()
|
driver := r.InstanceDRADriver()
|
||||||
|
|
@ -390,6 +404,17 @@ func (r unstructuredScalableResource) InstanceDRADriver() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r unstructuredScalableResource) readInfrastructureReferenceResource() (*unstructured.Unstructured, error) {
|
func (r unstructuredScalableResource) readInfrastructureReferenceResource() (*unstructured.Unstructured, error) {
|
||||||
|
// Cache w/ lazy loading of the infrastructure reference resource.
|
||||||
|
r.infraMutex.RLock()
|
||||||
|
if r.infraObj != nil {
|
||||||
|
defer r.infraMutex.RUnlock()
|
||||||
|
return r.infraObj, nil
|
||||||
|
}
|
||||||
|
r.infraMutex.RUnlock()
|
||||||
|
|
||||||
|
r.infraMutex.Lock()
|
||||||
|
defer r.infraMutex.Unlock()
|
||||||
|
|
||||||
obKind := r.unstructured.GetKind()
|
obKind := r.unstructured.GetKind()
|
||||||
obName := r.unstructured.GetName()
|
obName := r.unstructured.GetName()
|
||||||
|
|
||||||
|
|
@ -440,6 +465,8 @@ func (r unstructuredScalableResource) readInfrastructureReferenceResource() (*un
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
r.infraObj = infra
|
||||||
|
|
||||||
return infra, nil
|
return infra, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -477,6 +504,25 @@ func resourceCapacityFromInfrastructureObject(infraobj *unstructured.Unstructure
|
||||||
return capacity
|
return capacity
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func systemInfoFromInfrastructureObject(infraobj *unstructured.Unstructured) apiv1.NodeSystemInfo {
|
||||||
|
nsi := apiv1.NodeSystemInfo{}
|
||||||
|
infransi, found, err := unstructured.NestedStringMap(infraobj.Object, "status", "nodeInfo")
|
||||||
|
if !found || err != nil {
|
||||||
|
return nsi
|
||||||
|
}
|
||||||
|
|
||||||
|
for k, v := range infransi {
|
||||||
|
switch k {
|
||||||
|
case "architecture":
|
||||||
|
nsi.Architecture = v
|
||||||
|
case "operatingSystem":
|
||||||
|
nsi.OperatingSystem = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nsi
|
||||||
|
}
|
||||||
|
|
||||||
// adapted from https://github.com/kubernetes/kubernetes/blob/release-1.25/pkg/util/taints/taints.go#L39
|
// adapted from https://github.com/kubernetes/kubernetes/blob/release-1.25/pkg/util/taints/taints.go#L39
|
||||||
func parseTaint(st string) (apiv1.Taint, error) {
|
func parseTaint(st string) (apiv1.Taint, error) {
|
||||||
var taint apiv1.Taint
|
var taint apiv1.Taint
|
||||||
|
|
|
||||||
|
|
@ -33,9 +33,14 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
cpuStatusKey = "cpu"
|
cpuStatusKey = "cpu"
|
||||||
memoryStatusKey = "memory"
|
memoryStatusKey = "memory"
|
||||||
nvidiaGpuStatusKey = "nvidia.com/gpu"
|
nvidiaGpuStatusKey = "nvidia.com/gpu"
|
||||||
|
architectureStatusKey = "architecture"
|
||||||
|
operatingSystemStatusKey = "operatingSystem"
|
||||||
|
|
||||||
|
arm64 = "arm64"
|
||||||
|
linux = "linux"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSetSize(t *testing.T) {
|
func TestSetSize(t *testing.T) {
|
||||||
|
|
@ -127,6 +132,7 @@ func TestSetSize(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
))
|
))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -140,6 +146,7 @@ func TestSetSize(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
))
|
))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -244,11 +251,11 @@ func TestReplicas(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("MachineSet", func(t *testing.T) {
|
t.Run("MachineSet", func(t *testing.T) {
|
||||||
test(t, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), initialReplicas, nil, nil))
|
test(t, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), initialReplicas, nil, nil, nil))
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("MachineDeployment", func(t *testing.T) {
|
t.Run("MachineDeployment", func(t *testing.T) {
|
||||||
test(t, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), initialReplicas, nil, nil))
|
test(t, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), initialReplicas, nil, nil, nil))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -304,6 +311,7 @@ func TestSetSizeAndReplicas(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
))
|
))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -317,6 +325,7 @@ func TestSetSizeAndReplicas(t *testing.T) {
|
||||||
nodeGroupMaxSizeAnnotationKey: "10",
|
nodeGroupMaxSizeAnnotationKey: "10",
|
||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
|
nil,
|
||||||
))
|
))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -421,12 +430,12 @@ func TestAnnotations(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("MachineSet", func(t *testing.T) {
|
t.Run("MachineSet", func(t *testing.T) {
|
||||||
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, annotations, nil)
|
testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, annotations, nil, nil)
|
||||||
test(t, testConfig, testConfig.machineSet)
|
test(t, testConfig, testConfig.machineSet)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("MachineDeployment", func(t *testing.T) {
|
t.Run("MachineDeployment", func(t *testing.T) {
|
||||||
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, annotations, nil)
|
testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, annotations, nil, nil)
|
||||||
test(t, testConfig, testConfig.machineDeployment)
|
test(t, testConfig, testConfig.machineDeployment)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -531,7 +540,7 @@ func TestCanScaleFromZero(t *testing.T) {
|
||||||
for _, tc := range testConfigs {
|
for _, tc := range testConfigs {
|
||||||
testname := fmt.Sprintf("MachineSet %s", tc.name)
|
testname := fmt.Sprintf("MachineSet %s", tc.name)
|
||||||
t.Run(testname, func(t *testing.T) {
|
t.Run(testname, func(t *testing.T) {
|
||||||
msTestConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, tc.annotations, tc.capacity)
|
msTestConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, tc.annotations, tc.capacity, nil)
|
||||||
controller, stop := mustCreateTestController(t, msTestConfig)
|
controller, stop := mustCreateTestController(t, msTestConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
|
|
@ -552,7 +561,7 @@ func TestCanScaleFromZero(t *testing.T) {
|
||||||
for _, tc := range testConfigs {
|
for _, tc := range testConfigs {
|
||||||
testname := fmt.Sprintf("MachineDeployment %s", tc.name)
|
testname := fmt.Sprintf("MachineDeployment %s", tc.name)
|
||||||
t.Run(testname, func(t *testing.T) {
|
t.Run(testname, func(t *testing.T) {
|
||||||
msTestConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, tc.annotations, tc.capacity)
|
msTestConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, tc.annotations, tc.capacity, nil)
|
||||||
controller, stop := mustCreateTestController(t, msTestConfig)
|
controller, stop := mustCreateTestController(t, msTestConfig)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
|
|
@ -570,3 +579,98 @@ func TestCanScaleFromZero(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInstanceSystemInfo(t *testing.T) {
|
||||||
|
// use a constant capacity as that's necessary for the business logic to consider the resource scalable
|
||||||
|
capacity := map[string]string{
|
||||||
|
cpuStatusKey: "1",
|
||||||
|
memoryStatusKey: "4G",
|
||||||
|
}
|
||||||
|
testConfigs := []struct {
|
||||||
|
name string
|
||||||
|
nodeInfo map[string]string
|
||||||
|
expectedArch string
|
||||||
|
expectedOS string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
"with no architecture or operating system in machine template's status' nodeInfo, the system info is empty",
|
||||||
|
map[string]string{},
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"with architecture in machine template's status' nodeInfo, the system info is filled in the scalable resource",
|
||||||
|
map[string]string{
|
||||||
|
architectureStatusKey: arm64,
|
||||||
|
},
|
||||||
|
arm64,
|
||||||
|
"",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"with operating system in machine template's status' nodeInfo, the system info is filled in the scalable resource",
|
||||||
|
map[string]string{
|
||||||
|
operatingSystemStatusKey: linux,
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
linux,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"with architecture and operating system in machine template's status' nodeInfo, the system info is filled in the scalable resource",
|
||||||
|
map[string]string{
|
||||||
|
architectureStatusKey: arm64,
|
||||||
|
operatingSystemStatusKey: linux,
|
||||||
|
},
|
||||||
|
arm64,
|
||||||
|
linux,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testConfigs {
|
||||||
|
testname := fmt.Sprintf("MachineSet %s", tc.name)
|
||||||
|
t.Run(testname, func(t *testing.T) {
|
||||||
|
msTestConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, capacity, tc.nodeInfo)
|
||||||
|
controller, stop := mustCreateTestController(t, msTestConfig)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
testResource := msTestConfig.machineSet
|
||||||
|
|
||||||
|
sr, err := newUnstructuredScalableResource(controller, testResource)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sysInfo := sr.InstanceSystemInfo()
|
||||||
|
if sysInfo.Architecture != tc.expectedArch {
|
||||||
|
t.Errorf("expected architecture %s, got %s", tc.nodeInfo[architectureStatusKey], sysInfo.Architecture)
|
||||||
|
}
|
||||||
|
if sysInfo.OperatingSystem != tc.expectedOS {
|
||||||
|
t.Errorf("expected operating system %s, got %s", tc.nodeInfo[operatingSystemStatusKey], sysInfo.OperatingSystem)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testConfigs {
|
||||||
|
testname := fmt.Sprintf("MachineDeployment %s", tc.name)
|
||||||
|
t.Run(testname, func(t *testing.T) {
|
||||||
|
msTestConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, capacity, tc.nodeInfo)
|
||||||
|
controller, stop := mustCreateTestController(t, msTestConfig)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
testResource := msTestConfig.machineDeployment
|
||||||
|
|
||||||
|
sr, err := newUnstructuredScalableResource(controller, testResource)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sysInfo := sr.InstanceSystemInfo()
|
||||||
|
if sysInfo.Architecture != tc.expectedArch {
|
||||||
|
t.Errorf("expected architecture %s, got %s", tc.nodeInfo[architectureStatusKey], sysInfo.Architecture)
|
||||||
|
}
|
||||||
|
|
||||||
|
if sysInfo.OperatingSystem != tc.expectedOS {
|
||||||
|
t.Errorf("expected operating system %s, got %s", tc.nodeInfo[operatingSystemStatusKey], sysInfo.OperatingSystem)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue