Provide ScaleDownStatusProcessor with info about unremovable nodes

cluster-autoscaler/core/scale_down.go

@@ -363,6 +363,7 @@ type ScaleDown struct {
 	nodeUtilizationMap     map[string]simulator.UtilizationInfo
 	usageTracker           *simulator.UsageTracker
 	nodeDeletionTracker    *NodeDeletionTracker
+	unremovableNodeReasons map[string]*simulator.UnremovableNode
 }
 
 // NewScaleDown builds new ScaleDown object.
@@ -377,12 +378,14 @@ func NewScaleDown(context *context.AutoscalingContext, clusterStateRegistry *clu
 		usageTracker:           simulator.NewUsageTracker(),
 		unneededNodesList:      make([]*apiv1.Node, 0),
 		nodeDeletionTracker:    NewNodeDeletionTracker(),
+		unremovableNodeReasons: make(map[string]*simulator.UnremovableNode),
 	}
 }
 
 // CleanUp cleans up the internal ScaleDown state.
 func (sd *ScaleDown) CleanUp(timestamp time.Time) {
 	sd.usageTracker.CleanUp(timestamp.Add(-sd.context.ScaleDownUnneededTime))
+	sd.clearUnremovableNodeReasons()
 }
 
 // CleanUpUnneededNodes clears the list of unneeded nodes.
@@ -453,6 +456,7 @@ func (sd *ScaleDown) UpdateUnneededNodes(
 	for _, node := range scaleDownCandidates {
 		if unremovableTimestamp, found := sd.unremovableNodes[node.Name]; found {
 			if unremovableTimestamp.After(timestamp) {
+				sd.addUnremovableNodeReason(node, simulator.RecentlyUnremovable)
 				continue
 			}
 			delete(sd.unremovableNodes, node.Name)
@@ -473,18 +477,21 @@ func (sd *ScaleDown) UpdateUnneededNodes(
 		// and they have not been deleted.
 		if isNodeBeingDeleted(node, timestamp) {
 			klog.V(1).Infof("Skipping %s from delete consideration - the node is currently being deleted", node.Name)
+			sd.addUnremovableNodeReason(node, simulator.CurrentlyBeingDeleted)
 			continue
 		}
 
 		// Skip nodes marked with no scale down annotation
 		if hasNoScaleDownAnnotation(node) {
 			klog.V(1).Infof("Skipping %s from delete consideration - the node is marked as no scale down", node.Name)
+			sd.addUnremovableNodeReason(node, simulator.ScaleDownDisabledAnnotation)
 			continue
 		}
 
 		nodeInfo, found := nodeNameToNodeInfo[node.Name]
 		if !found {
 			klog.Errorf("Node info for %s not found", node.Name)
+			sd.addUnremovableNodeReason(node, simulator.UnexpectedError)
 			continue
 		}
 
@@ -497,6 +504,7 @@ func (sd *ScaleDown) UpdateUnneededNodes(
 
 		if !sd.isNodeBelowUtilzationThreshold(node, utilInfo) {
 			klog.V(4).Infof("Node %s is not suitable for removal - %s utilization too big (%f)", node.Name, utilInfo.ResourceName, utilInfo.Utilization)
+			sd.addUnremovableNodeReason(node, simulator.NotUnderutilized)
 			continue
 		}
 		currentlyUnneededNodes = append(currentlyUnneededNodes, node)
@@ -595,12 +603,22 @@ func (sd *ScaleDown) UpdateUnneededNodes(
 	// Add nodes to unremovable map
 	if len(unremovable) > 0 {
 		unremovableTimeout := timestamp.Add(sd.context.AutoscalingOptions.UnremovableNodeRecheckTimeout)
-		for _, node := range unremovable {
+		for _, unremovableNode := range unremovable {
-			sd.unremovableNodes[node.Name] = unremovableTimeout
+			sd.unremovableNodes[unremovableNode.Node.Name] = unremovableTimeout
+			sd.addUnremovableNode(unremovableNode)
 		}
 		klog.V(1).Infof("%v nodes found to be unremovable in simulation, will re-check them at %v", len(unremovable), unremovableTimeout)
 	}
 
+	// This method won't always check all nodes, so let's give a generic reason for all nodes that weren't checked.
+	for _, node := range scaleDownCandidates {
+		_, unremovableReasonProvided := sd.unremovableNodeReasons[node.Name]
+		_, unneeded := result[node.Name]
+		if !unneeded && !unremovableReasonProvided {
+			sd.addUnremovableNodeReason(node, simulator.NotUnneededOtherReason)
+		}
+	}
+
 	// Update state and metrics
 	sd.unneededNodesList = unneededNodesList
 	sd.unneededNodes = result
@@ -648,6 +666,18 @@ func (sd *ScaleDown) updateUnremovableNodes(nodes []*apiv1.Node) {
 	}
 }
 
+func (sd *ScaleDown) clearUnremovableNodeReasons() {
+	sd.unremovableNodeReasons = make(map[string]*simulator.UnremovableNode)
+}
+
+func (sd *ScaleDown) addUnremovableNodeReason(node *apiv1.Node, reason simulator.UnremovableReason) {
+	sd.unremovableNodeReasons[node.Name] = &simulator.UnremovableNode{Node: node, Reason: reason, BlockingPod: nil}
+}
+
+func (sd *ScaleDown) addUnremovableNode(unremovableNode *simulator.UnremovableNode) {
+	sd.unremovableNodeReasons[unremovableNode.Node.Name] = unremovableNode
+}
+
 // markSimulationError indicates a simulation error by clearing  relevant scale
 // down state and returning an appropriate error.
 func (sd *ScaleDown) markSimulationError(simulatorErr errors.AutoscalerError,
@@ -771,6 +801,7 @@ func (sd *ScaleDown) TryToScaleDown(allNodes []*apiv1.Node, pods []*apiv1.Pod, p
 			// Check if node is marked with no scale down annotation.
 			if hasNoScaleDownAnnotation(node) {
 				klog.V(4).Infof("Skipping %s - scale down disabled annotation found", node.Name)
+				sd.addUnremovableNodeReason(node, simulator.ScaleDownDisabledAnnotation)
 				continue
 			}
 
@@ -779,45 +810,53 @@ func (sd *ScaleDown) TryToScaleDown(allNodes []*apiv1.Node, pods []*apiv1.Pod, p
 
 			// Check how long the node was underutilized.
 			if ready && !val.Add(sd.context.ScaleDownUnneededTime).Before(currentTime) {
+				sd.addUnremovableNodeReason(node, simulator.NotUnneededLongEnough)
 				continue
 			}
 
 			// Unready nodes may be deleted after a different time than underutilized nodes.
 			if !ready && !val.Add(sd.context.ScaleDownUnreadyTime).Before(currentTime) {
+				sd.addUnremovableNodeReason(node, simulator.NotUnreadyLongEnough)
 				continue
 			}
 
 			nodeGroup, err := sd.context.CloudProvider.NodeGroupForNode(node)
 			if err != nil {
 				klog.Errorf("Error while checking node group for %s: %v", node.Name, err)
+				sd.addUnremovableNodeReason(node, simulator.UnexpectedError)
 				continue
 			}
 			if nodeGroup == nil || reflect.ValueOf(nodeGroup).IsNil() {
 				klog.V(4).Infof("Skipping %s - no node group config", node.Name)
+				sd.addUnremovableNodeReason(node, simulator.NotAutoscaled)
 				continue
 			}
 
 			size, found := nodeGroupSize[nodeGroup.Id()]
 			if !found {
 				klog.Errorf("Error while checking node group size %s: group size not found in cache", nodeGroup.Id())
+				sd.addUnremovableNodeReason(node, simulator.UnexpectedError)
 				continue
 			}
 
 			deletionsInProgress := sd.nodeDeletionTracker.GetDeletionsInProgress(nodeGroup.Id())
 			if size-deletionsInProgress <= nodeGroup.MinSize() {
 				klog.V(1).Infof("Skipping %s - node group min size reached", node.Name)
+				sd.addUnremovableNodeReason(node, simulator.NodeGroupMinSizeReached)
 				continue
 			}
 
 			scaleDownResourcesDelta, err := computeScaleDownResourcesDelta(sd.context.CloudProvider, node, nodeGroup, resourcesWithLimits)
 			if err != nil {
 				klog.Errorf("Error getting node resources: %v", err)
+				sd.addUnremovableNodeReason(node, simulator.UnexpectedError)
 				continue
 			}
 
 			checkResult := scaleDownResourcesLeft.checkScaleDownDeltaWithinLimits(scaleDownResourcesDelta)
 			if checkResult.exceeded {
 				klog.V(4).Infof("Skipping %s - minimal limit exceeded for %v", node.Name, checkResult.exceededResources)
+				sd.addUnremovableNodeReason(node, simulator.MinimalResourceLimitExceeded)
 				continue
 			}
 
@@ -857,7 +896,7 @@ func (sd *ScaleDown) TryToScaleDown(allNodes []*apiv1.Node, pods []*apiv1.Pod, p
 	// Only scheduled non expendable pods are taken into account and have to be moved.
 	nonExpendablePods := core_utils.FilterOutExpendablePods(pods, sd.context.ExpendablePodsPriorityCutoff)
 	// We look for only 1 node so new hints may be incomplete.
-	nodesToRemove, _, _, err := simulator.FindNodesToRemove(
+	nodesToRemove, unremovable, _, err := simulator.FindNodesToRemove(
 		candidates,
 		nodesWithoutMaster,
 		nonExpendablePods,
@@ -872,6 +911,10 @@ func (sd *ScaleDown) TryToScaleDown(allNodes []*apiv1.Node, pods []*apiv1.Pod, p
 		pdbs)
 	findNodesToRemoveDuration = time.Now().Sub(findNodesToRemoveStart)
 
+	for _, unremovableNode := range unremovable {
+		sd.addUnremovableNode(unremovableNode)
+	}
+
 	if err != nil {
 		scaleDownStatus.Result = status.ScaleDownError
 		return scaleDownStatus, err.AddPrefix("Find node to remove failed: ")

cluster-autoscaler/core/static_autoscaler.go

@@ -301,6 +301,7 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) errors.AutoscalerError
 			a.processors.ScaleUpStatusProcessor.Process(a.AutoscalingContext, scaleUpStatus)
 		}
 		if !scaleDownStatusProcessorAlreadyCalled && a.processors != nil && a.processors.ScaleDownStatusProcessor != nil {
+			scaleDownStatus.SetUnremovableNodesInfo(scaleDown.unremovableNodeReasons, scaleDown.nodeUtilizationMap, scaleDown.context.CloudProvider)
 			a.processors.ScaleDownStatusProcessor.Process(a.AutoscalingContext, scaleDownStatus)
 		}
 
@@ -534,6 +535,7 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) errors.AutoscalerError
 			}
 
 			if a.processors != nil && a.processors.ScaleDownStatusProcessor != nil {
+				scaleDownStatus.SetUnremovableNodesInfo(scaleDown.unremovableNodeReasons, scaleDown.nodeUtilizationMap, scaleDown.context.CloudProvider)
 				a.processors.ScaleDownStatusProcessor.Process(autoscalingContext, scaleDownStatus)
 				scaleDownStatusProcessorAlreadyCalled = true
 			}

cluster-autoscaler/processors/status/scale_down_status_processor.go

@@ -21,16 +21,55 @@ import (
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/autoscaler/cluster-autoscaler/context"
 	"k8s.io/autoscaler/cluster-autoscaler/simulator"
+	"k8s.io/autoscaler/cluster-autoscaler/utils/drain"
+	"k8s.io/klog"
 )
 
 // ScaleDownStatus represents the state of scale down.
 type ScaleDownStatus struct {
 	Result            ScaleDownResult
 	ScaledDownNodes   []*ScaleDownNode
+	UnremovableNodes  []*UnremovableNode
 	RemovedNodeGroups []cloudprovider.NodeGroup
 	NodeDeleteResults map[string]NodeDeleteResult
 }
 
+// SetUnremovableNodesInfo sets the status of nodes that were found to be unremovable.
+func (s *ScaleDownStatus) SetUnremovableNodesInfo(unremovableNodesMap map[string]*simulator.UnremovableNode, nodeUtilizationMap map[string]simulator.UtilizationInfo, cp cloudprovider.CloudProvider) {
+	s.UnremovableNodes = make([]*UnremovableNode, 0, len(unremovableNodesMap))
+
+	for _, unremovableNode := range unremovableNodesMap {
+		nodeGroup, err := cp.NodeGroupForNode(unremovableNode.Node)
+		if err != nil {
+			klog.Errorf("Couldn't find node group for unremovable node in cloud provider %s", unremovableNode.Node.Name)
+			continue
+		}
+
+		var utilInfoPtr *simulator.UtilizationInfo
+		if utilInfo, found := nodeUtilizationMap[unremovableNode.Node.Name]; found {
+			utilInfoPtr = &utilInfo
+			// It's okay if we don't find the util info, it's not computed for some unremovable nodes that are skipped early in the loop.
+		}
+
+		s.UnremovableNodes = append(s.UnremovableNodes, &UnremovableNode{
+			Node:        unremovableNode.Node,
+			NodeGroup:   nodeGroup,
+			UtilInfo:    utilInfoPtr,
+			Reason:      unremovableNode.Reason,
+			BlockingPod: unremovableNode.BlockingPod,
+		})
+	}
+}
+
+// UnremovableNode represents the state of a node that couldn't be removed.
+type UnremovableNode struct {
+	Node        *apiv1.Node
+	NodeGroup   cloudprovider.NodeGroup
+	UtilInfo    *simulator.UtilizationInfo
+	Reason      simulator.UnremovableReason
+	BlockingPod *drain.BlockingPod
+}
+
 // ScaleDownNode represents the state of a node that's being scaled down.
 type ScaleDownNode struct {
 	Node        *apiv1.Node

cluster-autoscaler/simulator/cluster.go

@@ -22,6 +22,7 @@ import (
 	"math/rand"
 	"time"
 
+	"k8s.io/autoscaler/cluster-autoscaler/utils/drain"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/errors"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/glogx"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/gpu"
@@ -57,6 +58,48 @@ type NodeToBeRemoved struct {
 	PodsToReschedule []*apiv1.Pod
 }
 
+// UnremovableNode represents a node that can't be removed by CA.
+type UnremovableNode struct {
+	Node        *apiv1.Node
+	Reason      UnremovableReason
+	BlockingPod *drain.BlockingPod
+}
+
+// UnremovableReason represents a reason why a node can't be removed by CA.
+type UnremovableReason int
+
+const (
+	// NoReason - sanity check, this should never be set explicitly. If this is found in the wild, it means that it was
+	// implicitly initialized and might indicate a bug.
+	NoReason UnremovableReason = iota
+	// ScaleDownDisabledAnnotation - node can't be removed because it has a "scale down disabled" annotation.
+	ScaleDownDisabledAnnotation
+	// NotAutoscaled - node can't be removed because it doesn't belong to an autoscaled node group.
+	NotAutoscaled
+	// NotUnneededLongEnough - node can't be removed because it wasn't unneeded for long enough.
+	NotUnneededLongEnough
+	// NotUnreadyLongEnough - node can't be removed because it wasn't unready for long enough.
+	NotUnreadyLongEnough
+	// NodeGroupMinSizeReached - node can't be removed because its node group is at its minimal size already.
+	NodeGroupMinSizeReached
+	// MinimalResourceLimitExceeded - node can't be removed because it would violate cluster-wide minimal resource limits.
+	MinimalResourceLimitExceeded
+	// CurrentlyBeingDeleted - node can't be removed because it's already in the process of being deleted.
+	CurrentlyBeingDeleted
+	// NotUnderutilized - node can't be removed because it's not underutilized.
+	NotUnderutilized
+	// NotUnneededOtherReason - node can't be removed because it's not marked as unneeded for other reasons (e.g. it wasn't inspected at all in a given autoscaler loop).
+	NotUnneededOtherReason
+	// RecentlyUnremovable - node can't be removed because it was recently found to be unremovable.
+	RecentlyUnremovable
+	// NoPlaceToMovePods - node can't be removed because there's no place to move its pods to.
+	NoPlaceToMovePods
+	// BlockedByPod - node can't be removed because a pod running on it can't be moved. The reason why should be in BlockingPod.
+	BlockedByPod
+	// UnexpectedError - node can't be removed because of an unexpected error.
+	UnexpectedError
+)
+
 // UtilizationInfo contains utilization information for a node.
 type UtilizationInfo struct {
 	CpuUtil float64
@@ -75,11 +118,11 @@ func FindNodesToRemove(candidates []*apiv1.Node, destinationNodes []*apiv1.Node,
 	fastCheck bool, oldHints map[string]string, usageTracker *UsageTracker,
 	timestamp time.Time,
 	podDisruptionBudgets []*policyv1.PodDisruptionBudget,
-) (nodesToRemove []NodeToBeRemoved, unremovableNodes []*apiv1.Node, podReschedulingHints map[string]string, finalError errors.AutoscalerError) {
+) (nodesToRemove []NodeToBeRemoved, unremovableNodes []*UnremovableNode, podReschedulingHints map[string]string, finalError errors.AutoscalerError) {
 
 	nodeNameToNodeInfo := scheduler_util.CreateNodeNameToInfoMap(pods, destinationNodes)
 	result := make([]NodeToBeRemoved, 0)
-	unremovable := make([]*apiv1.Node, 0)
+	unremovable := make([]*UnremovableNode, 0)
 
 	evaluationType := "Detailed evaluation"
 	if fastCheck {
@@ -92,24 +135,29 @@ candidateloop:
 		klog.V(2).Infof("%s: %s for removal", evaluationType, node.Name)
 
 		var podsToRemove []*apiv1.Pod
+		var blockingPod *drain.BlockingPod
 		var err error
 
 		if nodeInfo, found := nodeNameToNodeInfo[node.Name]; found {
 			if fastCheck {
-				podsToRemove, err = FastGetPodsToMove(nodeInfo, *skipNodesWithSystemPods, *skipNodesWithLocalStorage,
+				podsToRemove, blockingPod, err = FastGetPodsToMove(nodeInfo, *skipNodesWithSystemPods, *skipNodesWithLocalStorage,
 					podDisruptionBudgets)
 			} else {
-				podsToRemove, err = DetailedGetPodsForMove(nodeInfo, *skipNodesWithSystemPods, *skipNodesWithLocalStorage, listers, int32(*minReplicaCount),
+				podsToRemove, blockingPod, err = DetailedGetPodsForMove(nodeInfo, *skipNodesWithSystemPods, *skipNodesWithLocalStorage, listers, int32(*minReplicaCount),
 					podDisruptionBudgets)
 			}
 			if err != nil {
 				klog.V(2).Infof("%s: node %s cannot be removed: %v", evaluationType, node.Name, err)
-				unremovable = append(unremovable, node)
+				if blockingPod != nil {
+					unremovable = append(unremovable, &UnremovableNode{Node: node, Reason: BlockedByPod, BlockingPod: blockingPod})
+				} else {
+					unremovable = append(unremovable, &UnremovableNode{Node: node, Reason: UnexpectedError})
+				}
 				continue candidateloop
 			}
 		} else {
 			klog.V(2).Infof("%s: nodeInfo for %s not found", evaluationType, node.Name)
-			unremovable = append(unremovable, node)
+			unremovable = append(unremovable, &UnremovableNode{Node: node, Reason: UnexpectedError})
 			continue candidateloop
 		}
 		findProblems := findPlaceFor(node.Name, podsToRemove, destinationNodes, clusterSnapshot,
@@ -126,7 +174,7 @@ candidateloop:
 			}
 		} else {
 			klog.V(2).Infof("%s: node %s is not suitable for removal: %v", evaluationType, node.Name, findProblems)
-			unremovable = append(unremovable, node)
+			unremovable = append(unremovable, &UnremovableNode{Node: node, Reason: NoPlaceToMovePods})
 		}
 	}
 	return result, unremovable, newHints, nil
@@ -139,7 +187,7 @@ func FindEmptyNodesToRemove(candidates []*apiv1.Node, pods []*apiv1.Pod) []*apiv
 	for _, node := range candidates {
 		if nodeInfo, found := nodeNameToNodeInfo[node.Name]; found {
 			// Should block on all pods.
-			podsToRemove, err := FastGetPodsToMove(nodeInfo, true, true, nil)
+			podsToRemove, _, err := FastGetPodsToMove(nodeInfo, true, true, nil)
 			if err == nil && len(podsToRemove) == 0 {
 				result = append(result, node)
 			}

cluster-autoscaler/simulator/cluster_test.go

@@ -23,6 +23,7 @@ import (
 
 	apiv1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1beta1"
+	"k8s.io/autoscaler/cluster-autoscaler/utils/drain"
 	. "k8s.io/autoscaler/cluster-autoscaler/utils/test"
 	"k8s.io/kubernetes/pkg/kubelet/types"
 	schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
@@ -244,7 +245,7 @@ type findNodesToRemoveTestConfig struct {
 	candidates  []*apiv1.Node
 	allNodes    []*apiv1.Node
 	toRemove    []NodeToBeRemoved
-	unremovable []*apiv1.Node
+	unremovable []*UnremovableNode
 }
 
 func TestFindNodesToRemove(t *testing.T) {
@@ -299,7 +300,7 @@ func TestFindNodesToRemove(t *testing.T) {
 			candidates:  []*apiv1.Node{emptyNode},
 			allNodes:    []*apiv1.Node{emptyNode},
 			toRemove:    []NodeToBeRemoved{emptyNodeToRemove},
-			unremovable: []*apiv1.Node{},
+			unremovable: []*UnremovableNode{},
 		},
 		// just a drainable node, but nowhere for pods to go to
 		{
@@ -308,7 +309,7 @@ func TestFindNodesToRemove(t *testing.T) {
 			candidates:  []*apiv1.Node{drainableNode},
 			allNodes:    []*apiv1.Node{drainableNode},
 			toRemove:    []NodeToBeRemoved{},
-			unremovable: []*apiv1.Node{drainableNode},
+			unremovable: []*UnremovableNode{{Node: drainableNode, Reason: NoPlaceToMovePods}},
 		},
 		// drainable node, and a mostly empty node that can take its pods
 		{
@@ -317,7 +318,7 @@ func TestFindNodesToRemove(t *testing.T) {
 			candidates:  []*apiv1.Node{drainableNode, nonDrainableNode},
 			allNodes:    []*apiv1.Node{drainableNode, nonDrainableNode},
 			toRemove:    []NodeToBeRemoved{drainableNodeToRemove},
-			unremovable: []*apiv1.Node{nonDrainableNode},
+			unremovable: []*UnremovableNode{{Node: nonDrainableNode, Reason: BlockedByPod, BlockingPod: &drain.BlockingPod{Pod: pod3, Reason: drain.NotReplicated}}},
 		},
 		// drainable node, and a full node that cannot fit anymore pods
 		{
@@ -326,7 +327,7 @@ func TestFindNodesToRemove(t *testing.T) {
 			candidates:  []*apiv1.Node{drainableNode},
 			allNodes:    []*apiv1.Node{drainableNode, fullNode},
 			toRemove:    []NodeToBeRemoved{},
-			unremovable: []*apiv1.Node{drainableNode},
+			unremovable: []*UnremovableNode{{Node: drainableNode, Reason: NoPlaceToMovePods}},
 		},
 		// 4 nodes, 1 empty, 1 drainable
 		{
@@ -335,7 +336,7 @@ func TestFindNodesToRemove(t *testing.T) {
 			candidates:  []*apiv1.Node{emptyNode, drainableNode},
 			allNodes:    []*apiv1.Node{emptyNode, drainableNode, fullNode, nonDrainableNode},
 			toRemove:    []NodeToBeRemoved{emptyNodeToRemove, drainableNodeToRemove},
-			unremovable: []*apiv1.Node{},
+			unremovable: []*UnremovableNode{},
 		},
 	}
 

cluster-autoscaler/simulator/drain.go

@@ -35,8 +35,8 @@ import (
 // along with their pods (no abandoned pods with dangling created-by annotation). Useful for fast
 // checks.
 func FastGetPodsToMove(nodeInfo *schedulernodeinfo.NodeInfo, skipNodesWithSystemPods bool, skipNodesWithLocalStorage bool,
-	pdbs []*policyv1.PodDisruptionBudget) ([]*apiv1.Pod, error) {
+	pdbs []*policyv1.PodDisruptionBudget) ([]*apiv1.Pod, *drain.BlockingPod, error) {
-	pods, err := drain.GetPodsForDeletionOnNodeDrain(
+	pods, blockingPod, err := drain.GetPodsForDeletionOnNodeDrain(
 		nodeInfo.Pods(),
 		pdbs,
 		skipNodesWithSystemPods,
@@ -47,13 +47,13 @@ func FastGetPodsToMove(nodeInfo *schedulernodeinfo.NodeInfo, skipNodesWithSystem
 		time.Now())
 
 	if err != nil {
-		return pods, err
+		return pods, blockingPod, err
 	}
-	if err := checkPdbs(pods, pdbs); err != nil {
+	if pdbBlockingPod, err := checkPdbs(pods, pdbs); err != nil {
-		return []*apiv1.Pod{}, err
+		return []*apiv1.Pod{}, pdbBlockingPod, err
 	}
 
-	return pods, nil
+	return pods, nil, nil
 }
 
 // DetailedGetPodsForMove returns a list of pods that should be moved elsewhere if the node
@@ -62,8 +62,8 @@ func FastGetPodsToMove(nodeInfo *schedulernodeinfo.NodeInfo, skipNodesWithSystem
 // still exist.
 func DetailedGetPodsForMove(nodeInfo *schedulernodeinfo.NodeInfo, skipNodesWithSystemPods bool,
 	skipNodesWithLocalStorage bool, listers kube_util.ListerRegistry, minReplicaCount int32,
-	pdbs []*policyv1.PodDisruptionBudget) ([]*apiv1.Pod, error) {
+	pdbs []*policyv1.PodDisruptionBudget) ([]*apiv1.Pod, *drain.BlockingPod, error) {
-	pods, err := drain.GetPodsForDeletionOnNodeDrain(
+	pods, blockingPod, err := drain.GetPodsForDeletionOnNodeDrain(
 		nodeInfo.Pods(),
 		pdbs,
 		skipNodesWithSystemPods,
@@ -73,29 +73,29 @@ func DetailedGetPodsForMove(nodeInfo *schedulernodeinfo.NodeInfo, skipNodesWithS
 		minReplicaCount,
 		time.Now())
 	if err != nil {
-		return pods, err
+		return pods, blockingPod, err
 	}
-	if err := checkPdbs(pods, pdbs); err != nil {
+	if pdbBlockingPod, err := checkPdbs(pods, pdbs); err != nil {
-		return []*apiv1.Pod{}, err
+		return []*apiv1.Pod{}, pdbBlockingPod, err
 	}
 
-	return pods, nil
+	return pods, nil, nil
 }
 
-func checkPdbs(pods []*apiv1.Pod, pdbs []*policyv1.PodDisruptionBudget) error {
+func checkPdbs(pods []*apiv1.Pod, pdbs []*policyv1.PodDisruptionBudget) (*drain.BlockingPod, error) {
 	// TODO: make it more efficient.
 	for _, pdb := range pdbs {
 		selector, err := metav1.LabelSelectorAsSelector(pdb.Spec.Selector)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		for _, pod := range pods {
 			if pod.Namespace == pdb.Namespace && selector.Matches(labels.Set(pod.Labels)) {
 				if pdb.Status.DisruptionsAllowed < 1 {
-					return fmt.Errorf("not enough pod disruption budget to move %s/%s", pod.Namespace, pod.Name)
+					return &drain.BlockingPod{Pod: pod, Reason: drain.NotEnoughPdb}, fmt.Errorf("not enough pod disruption budget to move %s/%s", pod.Namespace, pod.Name)
 				}
 			}
 		}
 	}
-	return nil
+	return nil, nil
 }

cluster-autoscaler/simulator/drain_test.go

@@ -23,6 +23,7 @@ import (
 	policyv1 "k8s.io/api/policy/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/autoscaler/cluster-autoscaler/utils/drain"
 	. "k8s.io/autoscaler/cluster-autoscaler/utils/test"
 	"k8s.io/kubernetes/pkg/kubelet/types"
 	schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
@@ -39,8 +40,9 @@ func TestFastGetPodsToMove(t *testing.T) {
 			Namespace: "ns",
 		},
 	}
-	_, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod1), true, true, nil)
+	_, blockingPod, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod1), true, true, nil)
 	assert.Error(t, err)
+	assert.Equal(t, &drain.BlockingPod{Pod: pod1, Reason: drain.NotReplicated}, blockingPod)
 
 	// Replicated pod
 	pod2 := &apiv1.Pod{
@@ -50,8 +52,9 @@ func TestFastGetPodsToMove(t *testing.T) {
 			OwnerReferences: GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""),
 		},
 	}
-	r2, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod2), true, true, nil)
+	r2, blockingPod, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod2), true, true, nil)
 	assert.NoError(t, err)
+	assert.Nil(t, blockingPod)
 	assert.Equal(t, 1, len(r2))
 	assert.Equal(t, pod2, r2[0])
 
@@ -65,8 +68,9 @@ func TestFastGetPodsToMove(t *testing.T) {
 			},
 		},
 	}
-	r3, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod3), true, true, nil)
+	r3, blockingPod, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod3), true, true, nil)
 	assert.NoError(t, err)
+	assert.Nil(t, blockingPod)
 	assert.Equal(t, 0, len(r3))
 
 	// DaemonSet pod
@@ -77,8 +81,9 @@ func TestFastGetPodsToMove(t *testing.T) {
 			OwnerReferences: GenerateOwnerReferences("ds", "DaemonSet", "extensions/v1beta1", ""),
 		},
 	}
-	r4, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod2, pod3, pod4), true, true, nil)
+	r4, blockingPod, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod2, pod3, pod4), true, true, nil)
 	assert.NoError(t, err)
+	assert.Nil(t, blockingPod)
 	assert.Equal(t, 1, len(r4))
 	assert.Equal(t, pod2, r4[0])
 
@@ -90,8 +95,9 @@ func TestFastGetPodsToMove(t *testing.T) {
 			OwnerReferences: GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""),
 		},
 	}
-	_, err = FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod5), true, true, nil)
+	_, blockingPod, err = FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod5), true, true, nil)
 	assert.Error(t, err)
+	assert.Equal(t, &drain.BlockingPod{Pod: pod5, Reason: drain.UnmovableKubeSystemPod}, blockingPod)
 
 	// Local storage
 	pod6 := &apiv1.Pod{
@@ -110,8 +116,9 @@ func TestFastGetPodsToMove(t *testing.T) {
 			},
 		},
 	}
-	_, err = FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod6), true, true, nil)
+	_, blockingPod, err = FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod6), true, true, nil)
 	assert.Error(t, err)
+	assert.Equal(t, &drain.BlockingPod{Pod: pod6, Reason: drain.LocalStorageRequested}, blockingPod)
 
 	// Non-local storage
 	pod7 := &apiv1.Pod{
@@ -132,8 +139,9 @@ func TestFastGetPodsToMove(t *testing.T) {
 			},
 		},
 	}
-	r7, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod7), true, true, nil)
+	r7, blockingPod, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod7), true, true, nil)
 	assert.NoError(t, err)
+	assert.Nil(t, blockingPod)
 	assert.Equal(t, 1, len(r7))
 
 	// Pdb blocking
@@ -167,8 +175,9 @@ func TestFastGetPodsToMove(t *testing.T) {
 		},
 	}
 
-	_, err = FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod8), true, true, []*policyv1.PodDisruptionBudget{pdb8})
+	_, blockingPod, err = FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod8), true, true, []*policyv1.PodDisruptionBudget{pdb8})
 	assert.Error(t, err)
+	assert.Equal(t, &drain.BlockingPod{Pod: pod8, Reason: drain.NotEnoughPdb}, blockingPod)
 
 	// Pdb allowing
 	pod9 := &apiv1.Pod{
@@ -200,7 +209,8 @@ func TestFastGetPodsToMove(t *testing.T) {
 		},
 	}
 
-	r9, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod9), true, true, []*policyv1.PodDisruptionBudget{pdb9})
+	r9, blockingPod, err := FastGetPodsToMove(schedulernodeinfo.NewNodeInfo(pod9), true, true, []*policyv1.PodDisruptionBudget{pdb9})
 	assert.NoError(t, err)
+	assert.Nil(t, blockingPod)
 	assert.Equal(t, 1, len(r9))
 }

cluster-autoscaler/utils/drain/drain.go

@@ -25,7 +25,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-
 	kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes"
 	pod_util "k8s.io/autoscaler/cluster-autoscaler/utils/pod"
 )
@@ -41,6 +40,37 @@ const (
 	PodSafeToEvictKey = "cluster-autoscaler.kubernetes.io/safe-to-evict"
 )
 
+// BlockingPod represents a pod which is blocking the scale down of a node.
+type BlockingPod struct {
+	Pod    *apiv1.Pod
+	Reason BlockingPodReason
+}
+
+// BlockingPodReason represents a reason why a pod is blocking the scale down of a node.
+type BlockingPodReason int
+
+const (
+	// NoReason - sanity check, this should never be set explicitly. If this is found in the wild, it means that it was
+	// implicitly initialized and might indicate a bug.
+	NoReason BlockingPodReason = iota
+	// ControllerNotFound - pod is blocking scale down because its controller can't be found.
+	ControllerNotFound
+	// MinReplicasReached - pod is blocking scale down because its controller already has the minimum number of replicas.
+	MinReplicasReached
+	// NotReplicated - pod is blocking scale down because it's not replicated.
+	NotReplicated
+	// LocalStorageRequested - pod is blocking scale down because it requests local storage.
+	LocalStorageRequested
+	// NotSafeToEvictAnnotation - pod is blocking scale down because it has a "not safe to evict" annotation.
+	NotSafeToEvictAnnotation
+	// UnmovableKubeSystemPod - pod is blocking scale down because it's a non-daemonset, non-mirrored, non-pdb-assigned kube-system pod.
+	UnmovableKubeSystemPod
+	// NotEnoughPdb - pod is blocking scale down because it doesn't have enough PDB left.
+	NotEnoughPdb
+	// UnexpectedError - pod is blocking scale down because of an unexpected error.
+	UnexpectedError
+)
+
 // GetPodsForDeletionOnNodeDrain returns pods that should be deleted on node drain as well as some extra information
 // about possibly problematic pods (unreplicated and daemonsets).
 func GetPodsForDeletionOnNodeDrain(
@@ -51,7 +81,7 @@ func GetPodsForDeletionOnNodeDrain(
 	checkReferences bool, // Setting this to true requires client to be not-null.
 	listers kube_util.ListerRegistry,
 	minReplica int32,
-	currentTime time.Time) ([]*apiv1.Pod, error) {
+	currentTime time.Time) ([]*apiv1.Pod, *BlockingPod, error) {
 
 	pods := []*apiv1.Pod{}
 	// filter kube-system PDBs to avoid doing it for every kube-system pod
@@ -98,12 +128,12 @@ func GetPodsForDeletionOnNodeDrain(
 				// TODO: replace the minReplica check with pod disruption budget.
 				if err == nil && rc != nil {
 					if rc.Spec.Replicas != nil && *rc.Spec.Replicas < minReplica {
-						return []*apiv1.Pod{}, fmt.Errorf("replication controller for %s/%s has too few replicas spec: %d min: %d",
+						return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: MinReplicasReached}, fmt.Errorf("replication controller for %s/%s has too few replicas spec: %d min: %d",
 							pod.Namespace, pod.Name, rc.Spec.Replicas, minReplica)
 					}
 					replicated = true
 				} else {
-					return []*apiv1.Pod{}, fmt.Errorf("replication controller for %s/%s is not available, err: %v", pod.Namespace, pod.Name, err)
+					return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: ControllerNotFound}, fmt.Errorf("replication controller for %s/%s is not available, err: %v", pod.Namespace, pod.Name, err)
 				}
 			} else {
 				replicated = true
@@ -115,9 +145,9 @@ func GetPodsForDeletionOnNodeDrain(
 			if checkReferences && refKind == "DaemonSet" {
 				_, err := listers.DaemonSetLister().DaemonSets(controllerNamespace).Get(controllerRef.Name)
 				if apierrors.IsNotFound(err) {
-					return []*apiv1.Pod{}, fmt.Errorf("daemonset for %s/%s is not present, err: %v", pod.Namespace, pod.Name, err)
+					return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: ControllerNotFound}, fmt.Errorf("daemonset for %s/%s is not present, err: %v", pod.Namespace, pod.Name, err)
 				} else if err != nil {
-					return []*apiv1.Pod{}, fmt.Errorf("error when trying to get daemonset for %s/%s , err: %v", pod.Namespace, pod.Name, err)
+					return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: UnexpectedError}, fmt.Errorf("error when trying to get daemonset for %s/%s , err: %v", pod.Namespace, pod.Name, err)
 				}
 			}
 		} else if refKind == "Job" {
@@ -130,7 +160,7 @@ func GetPodsForDeletionOnNodeDrain(
 				if err == nil && job != nil {
 					replicated = true
 				} else {
-					return []*apiv1.Pod{}, fmt.Errorf("job for %s/%s is not available: err: %v", pod.Namespace, pod.Name, err)
+					return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: ControllerNotFound}, fmt.Errorf("job for %s/%s is not available: err: %v", pod.Namespace, pod.Name, err)
 				}
 			} else {
 				replicated = true
@@ -144,12 +174,12 @@ func GetPodsForDeletionOnNodeDrain(
 				// sophisticated than this
 				if err == nil && rs != nil {
 					if rs.Spec.Replicas != nil && *rs.Spec.Replicas < minReplica {
-						return []*apiv1.Pod{}, fmt.Errorf("replication controller for %s/%s has too few replicas spec: %d min: %d",
+						return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: MinReplicasReached}, fmt.Errorf("replication controller for %s/%s has too few replicas spec: %d min: %d",
 							pod.Namespace, pod.Name, rs.Spec.Replicas, minReplica)
 					}
 					replicated = true
 				} else {
-					return []*apiv1.Pod{}, fmt.Errorf("replication controller for %s/%s is not available, err: %v", pod.Namespace, pod.Name, err)
+					return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: ControllerNotFound}, fmt.Errorf("replication controller for %s/%s is not available, err: %v", pod.Namespace, pod.Name, err)
 				}
 			} else {
 				replicated = true
@@ -164,7 +194,7 @@ func GetPodsForDeletionOnNodeDrain(
 				if err == nil && ss != nil {
 					replicated = true
 				} else {
-					return []*apiv1.Pod{}, fmt.Errorf("statefulset for %s/%s is not available: err: %v", pod.Namespace, pod.Name, err)
+					return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: ControllerNotFound}, fmt.Errorf("statefulset for %s/%s is not available: err: %v", pod.Namespace, pod.Name, err)
 				}
 			} else {
 				replicated = true
@@ -176,27 +206,27 @@ func GetPodsForDeletionOnNodeDrain(
 
 		if !safeToEvict && !terminal {
 			if !replicated {
-				return []*apiv1.Pod{}, fmt.Errorf("%s/%s is not replicated", pod.Namespace, pod.Name)
+				return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: NotReplicated}, fmt.Errorf("%s/%s is not replicated", pod.Namespace, pod.Name)
 			}
 			if pod.Namespace == "kube-system" && skipNodesWithSystemPods {
 				hasPDB, err := checkKubeSystemPDBs(pod, kubeSystemPDBs)
 				if err != nil {
-					return []*apiv1.Pod{}, fmt.Errorf("error matching pods to pdbs: %v", err)
+					return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: UnexpectedError}, fmt.Errorf("error matching pods to pdbs: %v", err)
 				}
 				if !hasPDB {
-					return []*apiv1.Pod{}, fmt.Errorf("non-daemonset, non-mirrored, non-pdb-assigned kube-system pod present: %s", pod.Name)
+					return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: UnmovableKubeSystemPod}, fmt.Errorf("non-daemonset, non-mirrored, non-pdb-assigned kube-system pod present: %s", pod.Name)
 				}
 			}
 			if HasLocalStorage(pod) && skipNodesWithLocalStorage {
-				return []*apiv1.Pod{}, fmt.Errorf("pod with local storage present: %s", pod.Name)
+				return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: LocalStorageRequested}, fmt.Errorf("pod with local storage present: %s", pod.Name)
 			}
 			if hasNotSafeToEvictAnnotation(pod) {
-				return []*apiv1.Pod{}, fmt.Errorf("pod annotated as not safe to evict present: %s", pod.Name)
+				return []*apiv1.Pod{}, &BlockingPod{Pod: pod, Reason: NotSafeToEvictAnnotation}, fmt.Errorf("pod annotated as not safe to evict present: %s", pod.Name)
 			}
 		}
 		pods = append(pods, pod)
 	}
-	return pods, nil
+	return pods, nil, nil
 }
 
 // ControllerRef returns the OwnerReference to pod's controller.

cluster-autoscaler/utils/drain/drain_test.go

@@ -196,6 +196,7 @@ func TestDrain(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:            "bar",
 			Namespace:       "default",
+			OwnerReferences: GenerateOwnerReferences(rc.Name, "ReplicationController", "core/v1", ""),
 		},
 		Spec: apiv1.PodSpec{
 			NodeName: "node",
@@ -369,6 +370,7 @@ func TestDrain(t *testing.T) {
 		replicaSets       []*appsv1.ReplicaSet
 		expectFatal       bool
 		expectPods        []*apiv1.Pod
+		expectBlockingPod *BlockingPod
 	}{
 		{
 			description: "RC-managed pod",
@@ -430,13 +432,16 @@ func TestDrain(t *testing.T) {
 			pdbs:              []*policyv1.PodDisruptionBudget{},
 			expectFatal:       true,
 			expectPods:        []*apiv1.Pod{},
+			expectBlockingPod: &BlockingPod{Pod: nakedPod, Reason: NotReplicated},
 		},
 		{
 			description:       "pod with EmptyDir",
 			pods:              []*apiv1.Pod{emptydirPod},
 			pdbs:              []*policyv1.PodDisruptionBudget{},
+			rcs:               []*apiv1.ReplicationController{&rc},
 			expectFatal:       true,
 			expectPods:        []*apiv1.Pod{},
+			expectBlockingPod: &BlockingPod{Pod: emptydirPod, Reason: LocalStorageRequested},
 		},
 		{
 			description: "failed pod",
@@ -487,6 +492,7 @@ func TestDrain(t *testing.T) {
 			pdbs:              []*policyv1.PodDisruptionBudget{},
 			expectFatal:       true,
 			expectPods:        []*apiv1.Pod{},
+			expectBlockingPod: &BlockingPod{Pod: unsafeRcPod, Reason: NotSafeToEvictAnnotation},
 		},
 		{
 			description:       "Job-managed pod with PodSafeToEvict=false annotation",
@@ -495,6 +501,7 @@ func TestDrain(t *testing.T) {
 			rcs:               []*apiv1.ReplicationController{&rc},
 			expectFatal:       true,
 			expectPods:        []*apiv1.Pod{},
+			expectBlockingPod: &BlockingPod{Pod: unsafeJobPod, Reason: NotSafeToEvictAnnotation},
 		},
 		{
 			description: "empty PDB with RC-managed pod",
@@ -519,6 +526,7 @@ func TestDrain(t *testing.T) {
 			rcs:               []*apiv1.ReplicationController{&kubeSystemRc},
 			expectFatal:       true,
 			expectPods:        []*apiv1.Pod{},
+			expectBlockingPod: &BlockingPod{Pod: kubeSystemRcPod, Reason: UnmovableKubeSystemPod},
 		},
 		{
 			description: "kube-system PDB with default namespace pod",
@@ -535,6 +543,7 @@ func TestDrain(t *testing.T) {
 			rcs:               []*apiv1.ReplicationController{&kubeSystemRc},
 			expectFatal:       true,
 			expectPods:        []*apiv1.Pod{},
+			expectBlockingPod: &BlockingPod{Pod: kubeSystemRcPod, Reason: UnmovableKubeSystemPod},
 		},
 	}
 
@@ -560,15 +569,17 @@ func TestDrain(t *testing.T) {
 
 		registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, dsLister, rcLister, jobLister, rsLister, ssLister)
 
-		pods, err := GetPodsForDeletionOnNodeDrain(test.pods, test.pdbs, true, true, true, registry, 0, time.Now())
+		pods, blockingPod, err := GetPodsForDeletionOnNodeDrain(test.pods, test.pdbs, true, true, true, registry, 0, time.Now())
 
 		if test.expectFatal {
+			assert.Equal(t, test.expectBlockingPod, blockingPod)
 			if err == nil {
 				t.Fatalf("%s: unexpected non-error", test.description)
 			}
 		}
 
 		if !test.expectFatal {
+			assert.Nil(t, blockingPod)
 			if err != nil {
 				t.Fatalf("%s: error occurred: %v", test.description, err)
 			}