kubernetes
/
autoscaler
mirror of https://github.com/kubernetes/autoscaler.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changing deletion logic to rely on a new helper method in ClusterStateRegistry, and remove old complicated logic. Adjust the naming of the method for cloud instance deletion from NodeExists to HasInstance.

This commit is contained in:
Clint Fooken 2022-11-04 17:54:05 -07:00
parent ea7059f4c6
commit 08dfc7e20f
35 changed files with 116 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_provider.go
View File

@ -127,8 +127,8 @@ func (ali *aliCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.N
return ali.manager.GetAsgForInstance(instanceId)
}
// NodeExists returns whether node exists in this cloud provider
func (ali *aliCloudProvider) NodeExists(*apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (ali *aliCloudProvider) HasInstance(*apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go
View File

@ -120,8 +120,8 @@ func (aws *awsCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.N
}, nil
}
// NodeExists returns whether node exists in this cloud provider
func (aws *awsCloudProvider) NodeExists(*apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (aws *awsCloudProvider) HasInstance(*apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go
View File

@ -106,8 +106,8 @@ func (azure *AzureCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovid
return azure.azureManager.GetNodeGroupForInstance(ref)
}
// NodeExists returns whether node exists in this cloud provider
func (azure *AzureCloudProvider) NodeExists(*apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (azure *AzureCloudProvider) HasInstance(*apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/baiducloud/baiducloud_cloud_provider.go
View File

@ -180,8 +180,8 @@ func (baiducloud *baiducloudCloudProvider) NodeGroupForNode(node *apiv1.Node) (c
return asg, err
}
// NodeExists returns whether node exists in this cloud provider
func (baiducloud *baiducloudCloudProvider) NodeExists(*apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (baiducloud *baiducloudCloudProvider) HasInstance(*apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/bizflycloud/bizflycloud_cloud_provider.go
View File

@ -104,8 +104,8 @@ func (d *bizflycloudCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprov
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (d *bizflycloudCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (d *bizflycloudCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/brightbox/brightbox_cloud_provider.go
View File

@ -81,8 +81,8 @@ func (b *brightboxCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovid
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (b *brightboxCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (b *brightboxCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/cherryservers/cherry_cloud_provider.go
View File

@ -122,8 +122,8 @@ func (ccp *cherryCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovide
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (ccp *cherryCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (ccp *cherryCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/civo/civo_cloud_provider.go
View File

@ -99,8 +99,8 @@ func (d *civoCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.No
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (d *civoCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (d *civoCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

6
cluster-autoscaler/cloudprovider/cloud_provider.go
View File

@ -100,9 +100,9 @@ type CloudProvider interface {
// occurred. Must be implemented.
NodeGroupForNode(*apiv1.Node) (NodeGroup, error)
// NodeExists returns whether the node exists in cloud provider,
// true if the node is available, false if it has been deleted
NodeExists(*apiv1.Node) (bool, error)
// HasInstance returns whether the node has corresponding instance in cloud provider,
// true if the node has an instance, false if it no longer exists
HasInstance(*apiv1.Node) (bool, error)
// Pricing returns pricing model for this cloud provider or error if not available.
// Implementation optional.

6
cluster-autoscaler/cloudprovider/cloudstack/cloudstack_cloud_provider.go
View File

@ -68,8 +68,8 @@ func (provider *cloudStackCloudProvider) NodeGroupForNode(node *v1.Node) (cloudp
return provider.manager.clusterForNode(node)
}
// NodeExists returns whether node exists in this cloud provider
func (provider *cloudStackCloudProvider) NodeExists(node *v1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (provider *cloudStackCloudProvider) HasInstance(node *v1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}
@ -155,7 +155,7 @@ func BuildCloudStack(opts config.AutoscalingOptions, do cloudprovider.NodeGroupD
}
return &cloudStackCloudProvider{
manager: manager,
manager: manager,
resourceLimiter: rl,
}
}

4
cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go
View File

@ -81,8 +81,8 @@ func (p *provider) NodeGroupForNode(node *corev1.Node) (cloudprovider.NodeGroup,
return ng, nil
}
// NodeExists returns whether node exists in this cloud provider
func (p *provider) NodeExists(node *corev1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (p *provider) HasInstance(node *corev1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/digitalocean/digitalocean_cloud_provider.go
View File

@ -101,8 +101,8 @@ func (d *digitaloceanCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudpro
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (d *digitaloceanCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (d *digitaloceanCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/exoscale/exoscale_cloud_provider.go
View File

@ -131,8 +131,8 @@ func (e *exoscaleCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovide
return nodeGroup, nil
}
// NodeExists returns whether node exists in this cloud provider
func (e *exoscaleCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (e *exoscaleCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_cloud_provider.go
View File

@ -134,8 +134,8 @@ func (e *externalGrpcCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudpro
return ng, nil
}
// NodeExists returns whether node exists in this cloud provider
func (e *externalGrpcCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (e *externalGrpcCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/gce/gce_cloud_provider.go
View File

@ -101,8 +101,8 @@ func (gce *GceCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.N
return mig, err
}
// NodeExists returns whether node exists in this cloud provider
func (gce *GceCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (gce *GceCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go
View File

@ -99,8 +99,8 @@ func (d *HetznerCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider
return group, nil
}
// NodeExists returns whether node exists in this cloud provider
func (d *HetznerCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (d *HetznerCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud_cloud_provider.go
View File

@ -123,8 +123,8 @@ func (hcp *huaweicloudCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudpr
return hcp.cloudServiceManager.GetAsgForInstance(instanceID)
}
// NodeExists returns whether node exists in this cloud provider
func (hcp *huaweicloudCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (hcp *huaweicloudCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/ionoscloud/ionoscloud_cloud_provider.go
View File

@ -232,8 +232,8 @@ func (ic *IonosCloudCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprov
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (ic *IonosCloudCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (ic *IonosCloudCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/kamatera/kamatera_cloud_provider.go
View File

@ -70,8 +70,8 @@ func (k *kamateraCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovide
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (k *kamateraCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (k *kamateraCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/kubemark/kubemark_linux.go
View File

@ -139,8 +139,8 @@ func (kubemark *KubemarkCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloud
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (kubemark *KubemarkCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (kubemark *KubemarkCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/kubemark/kubemark_other.go
View File

@ -80,8 +80,8 @@ func (kubemark *KubemarkCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloud
return nil, cloudprovider.ErrNotImplemented
}
// NodeExists returns whether node exists in this cloud provider
func (kubemark *KubemarkCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (kubemark *KubemarkCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/linode/linode_cloud_provider.go
View File

@ -67,8 +67,8 @@ func (l *linodeCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (l *linodeCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (l *linodeCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/magnum/magnum_cloud_provider.go
View File

@ -135,8 +135,8 @@ func (mcp *magnumCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovide
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (mcp *magnumCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (mcp *magnumCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/mocks/CloudProvider.go
View File

@ -177,8 +177,8 @@ func (_m *CloudProvider) NodeGroupForNode(_a0 *v1.Node) (cloudprovider.NodeGroup
return r0, r1
}
// NodeExists provides a mock function with given fields:
func (_m *CloudProvider) NodeExists(_a0 *v1.Node) (bool, error) {
// HasInstance provides a mock function with given fields:
func (_m *CloudProvider) HasInstance(_a0 *v1.Node) (bool, error) {
ret := _m.Called(_a0)
var r0 bool

4
cluster-autoscaler/cloudprovider/oci/oci_cloud_provider.go
View File

@ -96,8 +96,8 @@ func (ocp *OciCloudProvider) NodeGroupForNode(n *apiv1.Node) (cloudprovider.Node
return ng, err
}
// NodeExists returns whether node exists in this cloud provider
func (ocp *OciCloudProvider) NodeExists(n *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (ocp *OciCloudProvider) HasInstance(n *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/ovhcloud/ovh_cloud_provider.go
View File

@ -151,8 +151,8 @@ func (provider *OVHCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovi
return ng, err
}
// NodeExists returns whether node exists in this cloud provider
func (provider *OVHCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (provider *OVHCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/packet/packet_cloud_provider.go
View File

@ -120,8 +120,8 @@ func (pcp *packetCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovide
return nil, fmt.Errorf("Could not find group for node: %s", node.Spec.ProviderID)
}
// NodeExists returns whether node exists in this cloud provider
func (pcp *packetCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (pcp *packetCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/rancher/rancher_provider.go
View File

@ -171,8 +171,8 @@ func (provider *RancherCloudProvider) NodeGroupForNode(node *corev1.Node) (cloud
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (provider *RancherCloudProvider) NodeExists(node *corev1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (provider *RancherCloudProvider) HasInstance(node *corev1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/scaleway/scaleway_cloud_provider.go
View File

@ -162,8 +162,8 @@ func (scw *scalewayCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovi
return scw.nodeGroupForNode(node)
}
// NodeExists returns whether node exists in this cloud provider
func (scw *scalewayCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (scw *scalewayCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

4
cluster-autoscaler/cloudprovider/tencentcloud/tencentcloud_cloud_provider.go
View File

@ -109,8 +109,8 @@ func (tencentcloud *tencentCloudProvider) NodeGroupForNode(node *apiv1.Node) (cl
return asg, nil
}
// NodeExists returns whether node exists in this cloud provider
func (tencentcloud *tencentCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (tencentcloud *tencentCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

18
cluster-autoscaler/cloudprovider/test/test_cloud_provider.go
View File

@ -41,8 +41,8 @@ type OnNodeGroupCreateFunc func(string) error
// OnNodeGroupDeleteFunc is a function called when a node group is deleted.
type OnNodeGroupDeleteFunc func(string) error
// NodeExists is a function called to determine if a node has been removed from the cloud provider.
type NodeExists func(string) (bool, error)
// HasInstance is a function called to determine if a node has been removed from the cloud provider.
type HasInstance func(string) (bool, error)
// TestCloudProvider is a dummy cloud provider to be used in tests.
type TestCloudProvider struct {
@ -53,7 +53,7 @@ type TestCloudProvider struct {
onScaleDown func(string, string) error
onNodeGroupCreate func(string) error
onNodeGroupDelete func(string) error
nodeExists func(string) (bool, error)
hasInstance func(string) (bool, error)
machineTypes []string
machineTemplates map[string]*schedulerframework.NodeInfo
priceModel cloudprovider.PricingModel
@ -90,13 +90,13 @@ func NewTestAutoprovisioningCloudProvider(onScaleUp OnScaleUpFunc, onScaleDown O
// NewTestNodeDeletionDetectionCloudProvider builds new TestCloudProvider with deletion detection support
func NewTestNodeDeletionDetectionCloudProvider(onScaleUp OnScaleUpFunc, onScaleDown OnScaleDownFunc,
deleted NodeExists) *TestCloudProvider {
hasInstance HasInstance) *TestCloudProvider {
return &TestCloudProvider{
nodes: make(map[string]string),
groups: make(map[string]cloudprovider.NodeGroup),
onScaleUp: onScaleUp,
onScaleDown: onScaleDown,
nodeExists: deleted,
hasInstance: hasInstance,
resourceLimiter: cloudprovider.NewResourceLimiter(make(map[string]int64), make(map[string]int64)),
}
}
@ -157,14 +157,14 @@ func (tcp *TestCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.
return group, nil
}
// NodeExists returns true if the node is available in cloud provider,
// HasInstance returns true if the node has corresponding instance in cloud provider,
// or ErrNotImplemented to fall back to taint-based node deletion in clusterstate
// readiness calculation.
func (tcp *TestCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
func (tcp *TestCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
tcp.Lock()
defer tcp.Unlock()
if tcp.nodeExists != nil {
return tcp.nodeExists(node.Name)
if tcp.hasInstance != nil {
return tcp.hasInstance(node.Name)
}
_, found := tcp.nodes[node.Name]
return found, nil

4
cluster-autoscaler/cloudprovider/vultr/vultr_cloud_provider.go
View File

@ -81,8 +81,8 @@ func (v *vultrCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.N
return nil, nil
}
// NodeExists returns whether node exists in this cloud provider
func (v *vultrCloudProvider) NodeExists(node *apiv1.Node) (bool, error) {
// HasInstance returns whether a given node has a corresponding instance in this cloud provider
func (v *vultrCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
return true, cloudprovider.ErrNotImplemented
}

78
cluster-autoscaler/clusterstate/clusterstate.go
View File

@ -298,7 +298,7 @@ func (csr *ClusterStateRegistry) UpdateNodes(nodes []*apiv1.Node, nodeInfosForGr
if err != nil {
return err
}
cloudProviderNodesRemoved := csr.getCloudProviderDeletedNodes(nodes, cloudProviderNodeInstances)
cloudProviderNodesRemoved := csr.getCloudProviderDeletedNodes(nodes)
notRegistered := getNotRegisteredNodes(nodes, cloudProviderNodeInstances, currentTime)
csr.Lock()
@ -675,7 +675,7 @@ func (csr *ClusterStateRegistry) GetUnregisteredNodes() []UnregisteredNode {
}
func (csr *ClusterStateRegistry) updateCloudProviderDeletedNodes(deletedNodes []*apiv1.Node) {
result := make(map[string]*apiv1.Node)
result := make(map[string]*apiv1.Node, len(deletedNodes)+len(csr.deletedNodes))
for _, deleted := range deletedNodes {
if prev, found := csr.deletedNodes[deleted.Name]; found {
result[deleted.Name] = prev
@ -988,70 +988,28 @@ func getNotRegisteredNodes(allNodes []*apiv1.Node, cloudProviderNodeInstances ma
}
// Calculates which of the registered nodes in Kubernetes that do not exist in cloud provider.
func (csr *ClusterStateRegistry) getCloudProviderDeletedNodes(allNodes []*apiv1.Node, cloudProviderNodeInstances map[string][]cloudprovider.Instance) []*apiv1.Node {
func (csr *ClusterStateRegistry) getCloudProviderDeletedNodes(allNodes []*apiv1.Node) []*apiv1.Node {
nodesRemoved := make([]*apiv1.Node, 0)
currentCloudInstances := make(map[string]string, 0)
registeredNodes := make(map[string]*apiv1.Node, 0)
if len(allNodes) > 0 {
_, err := csr.cloudProvider.NodeExists(allNodes[0])
// Check if the cloud provider implements nodeExists method
nodeExistsNotImplemented := errors.Is(err, cloudprovider.ErrNotImplemented)
if nodeExistsNotImplemented {
// Fall-back to taint-based node deletion
for _, node := range allNodes {
if deletetaint.HasToBeDeletedTaint(node) {
nodesRemoved = append(nodesRemoved, node)
}
}
} else {
for nodeGroupName, instances := range cloudProviderNodeInstances {
for _, instance := range instances {
currentCloudInstances[instance.Id] = nodeGroupName
}
}
for _, node := range allNodes {
registeredNodes[node.Name] = node
}
// Fill previously deleted nodes, if they are still registered in Kubernetes
for nodeName, node := range csr.deletedNodes {
// Safety check to prevent flagging Kubernetes nodes as deleted
// if the Cloud Provider instance is re-discovered
_, cloudProviderFound := currentCloudInstances[node.Name]
if _, found := registeredNodes[nodeName]; found && !cloudProviderFound {
// Confirm that node is deleted by cloud provider, instead of
// a not-autoscaled node
nodeExists, existsErr := csr.cloudProvider.NodeExists(node)
if existsErr == nil && !nodeExists {
nodesRemoved = append(nodesRemoved, node)
}
}
}
// Seek nodes that may have been deleted since last update
// cloudProviderNodeInstances are retrieved by nodeGroup,
// not autoscaled nodes will be excluded
for _, instances := range csr.cloudProviderNodeInstances {
for _, instance := range instances {
if _, found := currentCloudInstances[instance.Id]; !found {
// Check Kubernetes registered nodes for corresponding deleted
// Cloud Provider instance
if kubeNode, kubeNodeFound := registeredNodes[instance.Id]; kubeNodeFound {
// Confirm that node is deleted by cloud provider, instead of
// a not-autoscaled node
nodeExists, existsErr := csr.cloudProvider.NodeExists(kubeNode)
if existsErr == nil && !nodeExists {
nodesRemoved = append(nodesRemoved, kubeNode)
}
}
}
}
}
for _, node := range allNodes {
if !csr.hasCloudProviderInstance(node) {
nodesRemoved = append(nodesRemoved, node)
}
}
return nodesRemoved
}
func (csr *ClusterStateRegistry) hasCloudProviderInstance(node *apiv1.Node) bool {
exists, err := csr.cloudProvider.HasInstance(node)
if err == nil {
return exists
}
if !errors.Is(err, cloudprovider.ErrNotImplemented) {
klog.Warningf("Failed to check whether node has cloud instance for %s: %v", node.Name, err)
return exists
}
return !deletetaint.HasToBeDeletedTaint(node)
}
// GetAutoscaledNodesCount calculates and returns the actual and the target number of nodes
// belonging to autoscaled node groups in the cluster.
func (csr *ClusterStateRegistry) GetAutoscaledNodesCount() (currentSize, targetSize int) {

34
cluster-autoscaler/clusterstate/clusterstate_test.go
View File

@ -38,6 +38,18 @@ import (
"k8s.io/autoscaler/cluster-autoscaler/utils/backoff"
)
// GetCloudProviderDeletedNodes returns a list of all nodes removed from cloud provider but registered in Kubernetes.
func GetCloudProviderDeletedNodes(csr *ClusterStateRegistry) []*apiv1.Node {
csr.Lock()
defer csr.Unlock()
result := make([]*apiv1.Node, 0, len(csr.deletedNodes))
for _, deleted := range csr.deletedNodes {
result = append(result, deleted)
}
return result
}
func TestOKWithScaleUp(t *testing.T) {
now := time.Now()
@ -520,7 +532,7 @@ func TestUpcomingNodes(t *testing.T) {
}
func TestTaintBasedNodeDeletion(t *testing.T) {
// Create a new Cloud Provider that does not implement the NodeExists check
// Create a new Cloud Provider that does not implement the HasInstance check
// it will return the ErrNotImplemented error instead.
provider := testprovider.NewTestNodeDeletionDetectionCloudProvider(nil, nil,
func(string) (bool, error) { return false, cloudprovider.ErrNotImplemented })
@ -659,7 +671,7 @@ func TestCloudProviderDeletedNodes(t *testing.T) {
// Nodes are registered correctly between Kubernetes and cloud provider.
assert.NoError(t, err)
assert.Equal(t, 0, len(clusterstate.GetCloudProviderDeletedNodes()))
assert.Equal(t, 0, len(GetCloudProviderDeletedNodes(clusterstate)))
// The node was removed from Cloud Provider
// should be counted as Deleted by cluster state
@ -671,8 +683,8 @@ func TestCloudProviderDeletedNodes(t *testing.T) {
err = clusterstate.UpdateNodes([]*apiv1.Node{ng1_1, ng1_2, noNgNode}, nil, now)
assert.NoError(t, err)
assert.Equal(t, 1, len(clusterstate.GetCloudProviderDeletedNodes()))
assert.Equal(t, "ng1-2", clusterstate.GetCloudProviderDeletedNodes()[0].Name)
assert.Equal(t, 1, len(GetCloudProviderDeletedNodes(clusterstate)))
assert.Equal(t, "ng1-2", GetCloudProviderDeletedNodes(clusterstate)[0].Name)
assert.Equal(t, 1, clusterstate.GetClusterReadiness().Deleted)
// The node is removed from Kubernetes
@ -680,7 +692,7 @@ func TestCloudProviderDeletedNodes(t *testing.T) {
err = clusterstate.UpdateNodes([]*apiv1.Node{ng1_1, noNgNode}, nil, now)
assert.NoError(t, err)
assert.Equal(t, 0, len(clusterstate.GetCloudProviderDeletedNodes()))
assert.Equal(t, 0, len(GetCloudProviderDeletedNodes(clusterstate)))
// New Node is added afterwards
ng1_3 := BuildTestNode("ng1-3", 1000, 1000)
@ -692,7 +704,7 @@ func TestCloudProviderDeletedNodes(t *testing.T) {
err = clusterstate.UpdateNodes([]*apiv1.Node{ng1_1, ng1_3, noNgNode}, nil, now)
assert.NoError(t, err)
assert.Equal(t, 0, len(clusterstate.GetCloudProviderDeletedNodes()))
assert.Equal(t, 0, len(GetCloudProviderDeletedNodes(clusterstate)))
// Newly added node is removed from Cloud Provider
// should be counted as Deleted by cluster state
@ -704,8 +716,8 @@ func TestCloudProviderDeletedNodes(t *testing.T) {
err = clusterstate.UpdateNodes([]*apiv1.Node{ng1_1, noNgNode, ng1_3}, nil, now)
assert.NoError(t, err)
assert.Equal(t, 1, len(clusterstate.GetCloudProviderDeletedNodes()))
assert.Equal(t, "ng1-3", clusterstate.GetCloudProviderDeletedNodes()[0].Name)
assert.Equal(t, 1, len(GetCloudProviderDeletedNodes(clusterstate)))
assert.Equal(t, "ng1-3", GetCloudProviderDeletedNodes(clusterstate)[0].Name)
assert.Equal(t, 1, clusterstate.GetClusterReadiness().Deleted)
// Confirm that previously identified deleted Cloud Provider nodes are still included
@ -714,8 +726,8 @@ func TestCloudProviderDeletedNodes(t *testing.T) {
err = clusterstate.UpdateNodes([]*apiv1.Node{ng1_1, noNgNode, ng1_3}, nil, now)
assert.NoError(t, err)
assert.Equal(t, 1, len(clusterstate.GetCloudProviderDeletedNodes()))
assert.Equal(t, "ng1-3", clusterstate.GetCloudProviderDeletedNodes()[0].Name)
assert.Equal(t, 1, len(GetCloudProviderDeletedNodes(clusterstate)))
assert.Equal(t, "ng1-3", GetCloudProviderDeletedNodes(clusterstate)[0].Name)
assert.Equal(t, 1, clusterstate.GetClusterReadiness().Deleted)
// The node is removed from Kubernetes
@ -723,7 +735,7 @@ func TestCloudProviderDeletedNodes(t *testing.T) {
err = clusterstate.UpdateNodes([]*apiv1.Node{ng1_1, noNgNode}, nil, now)
assert.NoError(t, err)
assert.Equal(t, 0, len(clusterstate.GetCloudProviderDeletedNodes()))
assert.Equal(t, 0, len(GetCloudProviderDeletedNodes(clusterstate)))
}
func TestUpdateLastTransitionTimes(t *testing.T) {

4
cluster-autoscaler/core/static_autoscaler_test.go
View File

@ -1083,7 +1083,7 @@ func TestStaticAutoscalerInstanceCreationErrors(t *testing.T) {
}
return nil
}, nil)
provider.On("NodeExists", mock.Anything).Return(
provider.On("HasInstance", mock.Anything).Return(
func(node *apiv1.Node) bool {
return false
}, nil)
@ -1215,7 +1215,7 @@ func TestStaticAutoscalerInstanceCreationErrors(t *testing.T) {
provider = &mockprovider.CloudProvider{}
provider.On("NodeGroups").Return([]cloudprovider.NodeGroup{nodeGroupC})
provider.On("NodeGroupForNode", mock.Anything).Return(nil, nil)
provider.On("NodeExists", mock.Anything).Return(
provider.On("HasInstance", mock.Anything).Return(
func(node *apiv1.Node) bool {
return false
}, nil)