diff --git a/cluster-autoscaler/core/scale_up.go b/cluster-autoscaler/core/scale_up.go index 6873a8ec5f..17ec0276d8 100644 --- a/cluster-autoscaler/core/scale_up.go +++ b/cluster-autoscaler/core/scale_up.go @@ -29,7 +29,6 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/estimator" "k8s.io/autoscaler/cluster-autoscaler/expander" "k8s.io/autoscaler/cluster-autoscaler/metrics" - "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/glogx" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" @@ -56,8 +55,11 @@ func ScaleUp(context *context.AutoscalingContext, clusterStateRegistry *clusters loggingQuota := glogx.PodsLoggingQuota() + podsRemainUnschedulable := make(map[*apiv1.Pod]bool) + for _, pod := range unschedulablePods { glogx.V(1).UpTo(loggingQuota).Infof("Pod %s/%s is unschedulable", pod.Namespace, pod.Name) + podsRemainUnschedulable[pod] = true } glogx.V(1).Over(loggingQuota).Infof("%v other pods are also unschedulable", -loggingQuota.Left()) nodeInfos, err := GetNodeInfosForGroups(nodes, context.CloudProvider, context.ClientSet, @@ -93,7 +95,6 @@ func ScaleUp(context *context.AutoscalingContext, clusterStateRegistry *clusters glog.V(4).Infof("Upcoming %d nodes", len(upcomingNodes)) podsPassingPredicates := make(map[string][]*apiv1.Pod) - podsRemainUnschedulable := make(map[*apiv1.Pod]bool) expansionOptions := make([]expander.Option, 0) if context.AutoscalingOptions.NodeAutoprovisioningEnabled { @@ -144,17 +145,9 @@ func ScaleUp(context *context.AutoscalingContext, clusterStateRegistry *clusters Pods: make([]*apiv1.Pod, 0), } - for _, pod := range unschedulablePods { - err = context.PredicateChecker.CheckPredicates(pod, nil, nodeInfo, simulator.ReturnVerboseError) - if err == nil { - option.Pods = append(option.Pods, pod) - podsRemainUnschedulable[pod] = false - } else { - glog.V(2).Infof("Scale-up predicate failed: %v", err) - if _, exists := podsRemainUnschedulable[pod]; !exists { - podsRemainUnschedulable[pod] = true - } - } + option.Pods = FilterSchedulablePodsForNode(context, unschedulablePods, nodeGroup.Id(), nodeInfo) + for _, pod := range option.Pods { + podsRemainUnschedulable[pod] = false } passingPods := make([]*apiv1.Pod, len(option.Pods)) copy(passingPods, option.Pods) diff --git a/cluster-autoscaler/core/utils.go b/cluster-autoscaler/core/utils.go index 57313d3103..36fcd77f35 100644 --- a/cluster-autoscaler/core/utils.go +++ b/cluster-autoscaler/core/utils.go @@ -174,6 +174,36 @@ func FilterOutExpendablePods(pods []*apiv1.Pod, expendablePodsPriorityCutoff int return result } +// FilterSchedulablePodsForNode filters pods that can be scheduled on the given node. +func FilterSchedulablePodsForNode(context *context.AutoscalingContext, pods []*apiv1.Pod, nodeGroupId string, nodeInfo *schedulercache.NodeInfo) []*apiv1.Pod { + schedulablePods := []*apiv1.Pod{} + loggingQuota := glogx.PodsLoggingQuota() + podSchedulable := make(podSchedulableMap) + for _, pod := range pods { + schedulable, found := podSchedulable.get(pod) + if found { + if schedulable { + schedulablePods = append(schedulablePods, pod) + } else { + glogx.V(2).UpTo(loggingQuota).Infof("Pod %s can't be scheduled on %s. Used cached predicate check results", pod.Name, nodeGroupId) + } + } else { + err := context.PredicateChecker.CheckPredicates(pod, nil, nodeInfo, simulator.ReturnVerboseError) + if err == nil { + schedulable = true + podSchedulable.set(pod, true) + schedulablePods = append(schedulablePods, pod) + } else { + glog.V(2).Infof("Pod %s can't be scheduled on %s, predicate failed: %v", pod.Name, nodeGroupId, err) + schedulable = false + podSchedulable.set(pod, false) + } + } + } + glogx.V(2).Over(loggingQuota).Infof("%v other pods can't be scheduled on %s.", -loggingQuota.Left(), nodeGroupId) + return schedulablePods +} + // GetNodeInfosForGroups finds NodeInfos for all node groups used to manage the given nodes. It also returns a node group to sample node mapping. // TODO(mwielgus): This returns map keyed by url, while most code (including scheduler) uses node.Name for a key. // diff --git a/cluster-autoscaler/core/utils_test.go b/cluster-autoscaler/core/utils_test.go index 068502a5c7..46eace68ae 100644 --- a/cluster-autoscaler/core/utils_test.go +++ b/cluster-autoscaler/core/utils_test.go @@ -238,6 +238,52 @@ func TestFilterOutExpendablePods(t *testing.T) { assert.Equal(t, podWaitingForPreemption2, res[2]) } +func TestFilterSchedulablePodsForNode(t *testing.T) { + rc1 := apiv1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rc1", + Namespace: "default", + SelfLink: testapi.Default.SelfLink("replicationcontrollers", "rc"), + UID: "12345678-1234-1234-1234-123456789012", + }, + } + + rc2 := apiv1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rc2", + Namespace: "default", + SelfLink: testapi.Default.SelfLink("replicationcontrollers", "rc"), + UID: "12345678-1234-1234-1234-12345678901a", + }, + } + + p1 := BuildTestPod("p1", 1500, 200000) + p2_1 := BuildTestPod("p2_2", 3000, 200000) + p2_1.OwnerReferences = GenerateOwnerReferences(rc1.Name, "ReplicationController", "extensions/v1beta1", rc1.UID) + p2_2 := BuildTestPod("p2_2", 3000, 200000) + p2_2.OwnerReferences = GenerateOwnerReferences(rc1.Name, "ReplicationController", "extensions/v1beta1", rc1.UID) + p3_1 := BuildTestPod("p3", 100, 200000) + p3_1.OwnerReferences = GenerateOwnerReferences(rc2.Name, "ReplicationController", "extensions/v1beta1", rc2.UID) + p3_2 := BuildTestPod("p3", 100, 200000) + p3_2.OwnerReferences = GenerateOwnerReferences(rc2.Name, "ReplicationController", "extensions/v1beta1", rc2.UID) + unschedulablePods := []*apiv1.Pod{p1, p2_1, p2_2, p3_1, p3_2} + + tn := BuildTestNode("T1-abc", 2000, 2000000) + SetNodeReadyState(tn, true, time.Time{}) + tni := schedulercache.NewNodeInfo() + tni.SetNode(tn) + + context := &context.AutoscalingContext{ + PredicateChecker: simulator.NewTestPredicateChecker(), + } + + res := FilterSchedulablePodsForNode(context, unschedulablePods, "T1-abc", tni) + assert.Equal(t, 3, len(res)) + assert.Equal(t, p1, res[0]) + assert.Equal(t, p3_1, res[1]) + assert.Equal(t, p3_2, res[2]) +} + func TestGetNodeInfosForGroups(t *testing.T) { n1 := BuildTestNode("n1", 100, 1000) SetNodeReadyState(n1, true, time.Now())