Improve delete node mechanisms in cluster-autoscaler CAPI provider

This change adds a function to remove the annotations associated with
marking a node for deletion. It also adds logic to unmark a node in the
event that an error is returned after the node has been annotated but
before it has been removed. In the case where a node cannot be removed
(eg due to minimum size), the node is unmarked before we return from the
error condition.

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_machinedeployment.go

@@ -114,6 +114,10 @@ func (r machineDeploymentScalableResource) SetSize(nreplicas int32) error {
 	return updateErr
 }
 
+func (r machineDeploymentScalableResource) UnmarkMachineForDeletion(machine *Machine) error {
+	return unmarkMachineForDeletion(r.controller, machine)
+}
+
 func (r machineDeploymentScalableResource) MarkMachineForDeletion(machine *Machine) error {
 	u, err := r.controller.dynamicclient.Resource(*r.controller.machineResource).Namespace(machine.Namespace).Get(context.TODO(), machine.Name, metav1.GetOptions{})
 

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_machineset.go

@@ -123,6 +123,10 @@ func (r machineSetScalableResource) MarkMachineForDeletion(machine *Machine) err
 	return updateErr
 }
 
+func (r machineSetScalableResource) UnmarkMachineForDeletion(machine *Machine) error {
+	return unmarkMachineForDeletion(r.controller, machine)
+}
+
 func newMachineSetScalableResource(controller *machineController, machineSet *MachineSet) (*machineSetScalableResource, error) {
 	minSize, maxSize, err := parseScalingBounds(machineSet.Annotations)
 	if err != nil {

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go

@@ -160,6 +160,7 @@ func (ng *nodegroup) DeleteNodes(nodes []*corev1.Node) error {
 		}
 
 		if err := ng.scalableResource.SetSize(replicas - 1); err != nil {
+			nodeGroup.scalableResource.UnmarkMachineForDeletion(machine)
 			return err
 		}
 

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_scalableresource.go

@@ -16,6 +16,13 @@ limitations under the License.
 
 package clusterapi
 
+import (
+	"context"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
 // scalableResource is a resource that can be scaled up and down by
 // adjusting its replica count field.
 type scalableResource interface {
@@ -46,4 +53,27 @@ type scalableResource interface {
 
 	// MarkMachineForDeletion marks machine for deletion
 	MarkMachineForDeletion(machine *Machine) error
+
+	// UnmarkMachineForDeletion unmarks machine for deletion
+	UnmarkMachineForDeletion(machine *Machine) error
+}
+
+func unmarkMachineForDeletion(controller *machineController, machine *Machine) error {
+	u, err := controller.dynamicclient.Resource(*controller.machineResource).Namespace(machine.Namespace).Get(context.TODO(), machine.Name, metav1.GetOptions{})
+
+	if err != nil {
+		return err
+	}
+	if u == nil {
+		return fmt.Errorf("unknown machine %s", machine.Name)
+	}
+
+	annotations := u.GetAnnotations()
+	if _, ok := annotations[machineDeleteAnnotationKey]; ok {
+		delete(annotations, machineDeleteAnnotationKey)
+		u.SetAnnotations(annotations)
+		_, updateErr := controller.dynamicclient.Resource(*controller.machineResource).Namespace(u.GetNamespace()).Update(context.TODO(), u, metav1.UpdateOptions{})
+		return updateErr
+	}
+	return nil
 }