Execute predicates only for similar pods.

cluster-autoscaler/core/scale_up.go

@@ -29,7 +29,6 @@ import (
 	"k8s.io/autoscaler/cluster-autoscaler/estimator"
 	"k8s.io/autoscaler/cluster-autoscaler/expander"
 	"k8s.io/autoscaler/cluster-autoscaler/metrics"
-	"k8s.io/autoscaler/cluster-autoscaler/simulator"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/errors"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/glogx"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/gpu"
@@ -56,8 +55,11 @@ func ScaleUp(context *context.AutoscalingContext, clusterStateRegistry *clusters
 
 	loggingQuota := glogx.PodsLoggingQuota()
 
+	podsRemainUnschedulable := make(map[*apiv1.Pod]bool)
+
 	for _, pod := range unschedulablePods {
 		glogx.V(1).UpTo(loggingQuota).Infof("Pod %s/%s is unschedulable", pod.Namespace, pod.Name)
+		podsRemainUnschedulable[pod] = true
 	}
 	glogx.V(1).Over(loggingQuota).Infof("%v other pods are also unschedulable", -loggingQuota.Left())
 	nodeInfos, err := GetNodeInfosForGroups(nodes, context.CloudProvider, context.ClientSet,
@@ -93,7 +95,6 @@ func ScaleUp(context *context.AutoscalingContext, clusterStateRegistry *clusters
 	glog.V(4).Infof("Upcoming %d nodes", len(upcomingNodes))
 
 	podsPassingPredicates := make(map[string][]*apiv1.Pod)
-	podsRemainUnschedulable := make(map[*apiv1.Pod]bool)
 	expansionOptions := make([]expander.Option, 0)
 
 	if context.AutoscalingOptions.NodeAutoprovisioningEnabled {
@@ -144,17 +145,9 @@ func ScaleUp(context *context.AutoscalingContext, clusterStateRegistry *clusters
 			Pods:      make([]*apiv1.Pod, 0),
 		}
 
-		for _, pod := range unschedulablePods {
-			err = context.PredicateChecker.CheckPredicates(pod, nil, nodeInfo, simulator.ReturnVerboseError)
-			if err == nil {
-				option.Pods = append(option.Pods, pod)
-				podsRemainUnschedulable[pod] = false
-			} else {
-				glog.V(2).Infof("Scale-up predicate failed: %v", err)
-				if _, exists := podsRemainUnschedulable[pod]; !exists {
-					podsRemainUnschedulable[pod] = true
-				}
-			}
+		option.Pods = FilterSchedulablePodsForNode(context, unschedulablePods, nodeGroup.Id(), nodeInfo)
+		for _, pod := range option.Pods {
+			podsRemainUnschedulable[pod] = false
 		}
 		passingPods := make([]*apiv1.Pod, len(option.Pods))
 		copy(passingPods, option.Pods)

cluster-autoscaler/core/utils.go

@@ -174,6 +174,36 @@ func FilterOutExpendablePods(pods []*apiv1.Pod, expendablePodsPriorityCutoff int
 	return result
 }
 
+// FilterSchedulablePodsForNode filters pods that can be scheduled on the given node.
+func FilterSchedulablePodsForNode(context *context.AutoscalingContext, pods []*apiv1.Pod, nodeGroupId string, nodeInfo *schedulercache.NodeInfo) []*apiv1.Pod {
+	schedulablePods := []*apiv1.Pod{}
+	loggingQuota := glogx.PodsLoggingQuota()
+	podSchedulable := make(podSchedulableMap)
+	for _, pod := range pods {
+		schedulable, found := podSchedulable.get(pod)
+		if found {
+			if schedulable {
+				schedulablePods = append(schedulablePods, pod)
+			} else {
+				glogx.V(2).UpTo(loggingQuota).Infof("Pod %s can't be scheduled on %s. Used cached predicate check results", pod.Name, nodeGroupId)
+			}
+		} else {
+			err := context.PredicateChecker.CheckPredicates(pod, nil, nodeInfo, simulator.ReturnVerboseError)
+			if err == nil {
+				schedulable = true
+				podSchedulable.set(pod, true)
+				schedulablePods = append(schedulablePods, pod)
+			} else {
+				glog.V(2).Infof("Pod %s can't be scheduled on %s, predicate failed: %v", pod.Name, nodeGroupId, err)
+				schedulable = false
+				podSchedulable.set(pod, false)
+			}
+		}
+	}
+	glogx.V(2).Over(loggingQuota).Infof("%v other pods can't be scheduled on %s.", -loggingQuota.Left(), nodeGroupId)
+	return schedulablePods
+}
+
 // GetNodeInfosForGroups finds NodeInfos for all node groups used to manage the given nodes. It also returns a node group to sample node mapping.
 // TODO(mwielgus): This returns map keyed by url, while most code (including scheduler) uses node.Name for a key.
 //

cluster-autoscaler/core/utils_test.go

@@ -238,6 +238,52 @@ func TestFilterOutExpendablePods(t *testing.T) {
 	assert.Equal(t, podWaitingForPreemption2, res[2])
 }
 
+func TestFilterSchedulablePodsForNode(t *testing.T) {
+	rc1 := apiv1.ReplicationController{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "rc1",
+			Namespace: "default",
+			SelfLink:  testapi.Default.SelfLink("replicationcontrollers", "rc"),
+			UID:       "12345678-1234-1234-1234-123456789012",
+		},
+	}
+
+	rc2 := apiv1.ReplicationController{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "rc2",
+			Namespace: "default",
+			SelfLink:  testapi.Default.SelfLink("replicationcontrollers", "rc"),
+			UID:       "12345678-1234-1234-1234-12345678901a",
+		},
+	}
+
+	p1 := BuildTestPod("p1", 1500, 200000)
+	p2_1 := BuildTestPod("p2_2", 3000, 200000)
+	p2_1.OwnerReferences = GenerateOwnerReferences(rc1.Name, "ReplicationController", "extensions/v1beta1", rc1.UID)
+	p2_2 := BuildTestPod("p2_2", 3000, 200000)
+	p2_2.OwnerReferences = GenerateOwnerReferences(rc1.Name, "ReplicationController", "extensions/v1beta1", rc1.UID)
+	p3_1 := BuildTestPod("p3", 100, 200000)
+	p3_1.OwnerReferences = GenerateOwnerReferences(rc2.Name, "ReplicationController", "extensions/v1beta1", rc2.UID)
+	p3_2 := BuildTestPod("p3", 100, 200000)
+	p3_2.OwnerReferences = GenerateOwnerReferences(rc2.Name, "ReplicationController", "extensions/v1beta1", rc2.UID)
+	unschedulablePods := []*apiv1.Pod{p1, p2_1, p2_2, p3_1, p3_2}
+
+	tn := BuildTestNode("T1-abc", 2000, 2000000)
+	SetNodeReadyState(tn, true, time.Time{})
+	tni := schedulercache.NewNodeInfo()
+	tni.SetNode(tn)
+
+	context := &context.AutoscalingContext{
+		PredicateChecker: simulator.NewTestPredicateChecker(),
+	}
+
+	res := FilterSchedulablePodsForNode(context, unschedulablePods, "T1-abc", tni)
+	assert.Equal(t, 3, len(res))
+	assert.Equal(t, p1, res[0])
+	assert.Equal(t, p3_1, res[1])
+	assert.Equal(t, p3_2, res[2])
+}
+
 func TestGetNodeInfosForGroups(t *testing.T) {
 	n1 := BuildTestNode("n1", 100, 1000)
 	SetNodeReadyState(n1, true, time.Now())