305 lines
9.2 KiB
Go
305 lines
9.2 KiB
Go
/*
|
|
Copyright 2018 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package predicates
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"k8s.io/klog"
|
|
"k8s.io/kubernetes/pkg/scheduler/algorithm"
|
|
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
|
|
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
|
|
|
|
"volcano.sh/volcano/pkg/scheduler/api"
|
|
"volcano.sh/volcano/pkg/scheduler/framework"
|
|
"volcano.sh/volcano/pkg/scheduler/plugins/util"
|
|
)
|
|
|
|
const (
|
|
// PluginName indicates name of volcano scheduler plugin.
|
|
PluginName = "predicates"
|
|
|
|
// MemoryPressurePredicate is the key for enabling Memory Pressure Predicate in YAML
|
|
MemoryPressurePredicate = "predicate.MemoryPressureEnable"
|
|
// DiskPressurePredicate is the key for enabling Disk Pressure Predicate in YAML
|
|
DiskPressurePredicate = "predicate.DiskPressureEnable"
|
|
// PIDPressurePredicate is the key for enabling PID Pressure Predicate in YAML
|
|
PIDPressurePredicate = "predicate.PIDPressureEnable"
|
|
)
|
|
|
|
type predicatesPlugin struct {
|
|
// Arguments given for the plugin
|
|
pluginArguments framework.Arguments
|
|
}
|
|
|
|
// New return predicate plugin
|
|
func New(arguments framework.Arguments) framework.Plugin {
|
|
return &predicatesPlugin{pluginArguments: arguments}
|
|
}
|
|
|
|
func (pp *predicatesPlugin) Name() string {
|
|
return PluginName
|
|
}
|
|
|
|
func formatReason(reasons []predicates.PredicateFailureReason) string {
|
|
reasonStrings := []string{}
|
|
for _, v := range reasons {
|
|
reasonStrings = append(reasonStrings, fmt.Sprintf("%v", v.GetReason()))
|
|
}
|
|
|
|
return strings.Join(reasonStrings, ", ")
|
|
}
|
|
|
|
type predicateEnable struct {
|
|
memoryPressureEnable bool
|
|
diskPressureEnable bool
|
|
pidPressureEnable bool
|
|
}
|
|
|
|
func enablePredicate(args framework.Arguments) predicateEnable {
|
|
|
|
/*
|
|
User Should give predicatesEnable in this format(predicate.MemoryPressureEnable, predicate.DiskPressureEnable, predicate.PIDPressureEnable.
|
|
Currently supported only for MemoryPressure, DiskPressure, PIDPressure predicate checks.
|
|
|
|
actions: "reclaim, allocate, backfill, preempt"
|
|
tiers:
|
|
- plugins:
|
|
- name: priority
|
|
- name: gang
|
|
- name: conformance
|
|
- plugins:
|
|
- name: drf
|
|
- name: predicates
|
|
arguments:
|
|
predicate.MemoryPressureEnable: true
|
|
predicate.DiskPressureEnable: true
|
|
predicate.PIDPressureEnable: true
|
|
- name: proportion
|
|
- name: nodeorder
|
|
*/
|
|
|
|
predicate := predicateEnable{
|
|
memoryPressureEnable: false,
|
|
diskPressureEnable: false,
|
|
pidPressureEnable: false,
|
|
}
|
|
|
|
// Checks whether predicate.MemoryPressureEnable is provided or not, if given, modifies the value in predicateEnable struct.
|
|
args.GetBool(&predicate.memoryPressureEnable, MemoryPressurePredicate)
|
|
|
|
// Checks whether predicate.DiskPressureEnable is provided or not, if given, modifies the value in predicateEnable struct.
|
|
args.GetBool(&predicate.diskPressureEnable, DiskPressurePredicate)
|
|
|
|
// Checks whether predicate.PIDPressureEnable is provided or not, if given, modifies the value in predicateEnable struct.
|
|
args.GetBool(&predicate.pidPressureEnable, PIDPressurePredicate)
|
|
|
|
return predicate
|
|
}
|
|
|
|
func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
|
|
var nodeMap map[string]*schedulernodeinfo.NodeInfo
|
|
|
|
pl := util.NewPodLister(ssn)
|
|
|
|
nodeMap, _ = util.GenerateNodeMapAndSlice(ssn.Nodes)
|
|
|
|
// Register event handlers to update task info in PodLister & nodeMap
|
|
ssn.AddEventHandler(&framework.EventHandler{
|
|
AllocateFunc: func(event *framework.Event) {
|
|
pod := pl.UpdateTask(event.Task, event.Task.NodeName)
|
|
|
|
nodeName := event.Task.NodeName
|
|
node, found := nodeMap[nodeName]
|
|
if !found {
|
|
klog.Warningf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName)
|
|
} else {
|
|
node.AddPod(pod)
|
|
klog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName)
|
|
}
|
|
},
|
|
DeallocateFunc: func(event *framework.Event) {
|
|
pod := pl.UpdateTask(event.Task, "")
|
|
|
|
nodeName := event.Task.NodeName
|
|
node, found := nodeMap[nodeName]
|
|
if !found {
|
|
klog.Warningf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName)
|
|
} else {
|
|
node.RemovePod(pod)
|
|
klog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName)
|
|
}
|
|
},
|
|
})
|
|
|
|
ni := &util.CachedNodeInfo{
|
|
Session: ssn,
|
|
}
|
|
|
|
predicate := enablePredicate(pp.pluginArguments)
|
|
|
|
ssn.AddPredicateFn(pp.Name(), func(task *api.TaskInfo, node *api.NodeInfo) error {
|
|
nodeInfo, found := nodeMap[node.Name]
|
|
if !found {
|
|
nodeInfo = schedulernodeinfo.NewNodeInfo(node.Pods()...)
|
|
nodeInfo.SetNode(node.Node)
|
|
klog.Warningf("predicates, generate node info for %s at PredicateFn is unexpected", node.Name)
|
|
}
|
|
|
|
if node.Allocatable.MaxTaskNum <= len(nodeInfo.Pods()) {
|
|
klog.V(4).Infof("NodePodNumber predicates Task <%s/%s> on Node <%s> failed",
|
|
task.Namespace, task.Name, node.Name)
|
|
return api.NewFitError(task, node, api.NodePodNumberExceeded)
|
|
}
|
|
|
|
// CheckNodeCondition Predicate
|
|
fit, reasons, err := predicates.CheckNodeConditionPredicate(task.Pod, nil, nodeInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
klog.V(4).Infof("CheckNodeCondition predicates Task <%s/%s> on Node <%s>: fit %t, err %v",
|
|
task.Namespace, task.Name, node.Name, fit, err)
|
|
|
|
if !fit {
|
|
return api.NewFitErrorByReasons(task, node, reasons...)
|
|
}
|
|
|
|
// CheckNodeUnschedulable Predicate
|
|
fit, reasons, err = predicates.CheckNodeUnschedulablePredicate(task.Pod, nil, nodeInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
klog.V(4).Infof("CheckNodeUnschedulable Predicate Task <%s/%s> on Node <%s>: fit %t, err %v",
|
|
task.Namespace, task.Name, node.Name, fit, err)
|
|
|
|
if !fit {
|
|
return api.NewFitErrorByReasons(task, node, reasons...)
|
|
}
|
|
|
|
// NodeSelector Predicate
|
|
fit, reasons, err = predicates.PodMatchNodeSelector(task.Pod, nil, nodeInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
klog.V(4).Infof("NodeSelect predicates Task <%s/%s> on Node <%s>: fit %t, err %v",
|
|
task.Namespace, task.Name, node.Name, fit, err)
|
|
|
|
if !fit {
|
|
return api.NewFitErrorByReasons(task, node, reasons...)
|
|
}
|
|
|
|
// HostPorts Predicate
|
|
fit, reasons, err = predicates.PodFitsHostPorts(task.Pod, nil, nodeInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
klog.V(4).Infof("HostPorts predicates Task <%s/%s> on Node <%s>: fit %t, err %v",
|
|
task.Namespace, task.Name, node.Name, fit, err)
|
|
|
|
if !fit {
|
|
return api.NewFitErrorByReasons(task, node, reasons...)
|
|
}
|
|
|
|
// Toleration/Taint Predicate
|
|
fit, reasons, err = predicates.PodToleratesNodeTaints(task.Pod, nil, nodeInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
klog.V(4).Infof("Toleration/Taint predicates Task <%s/%s> on Node <%s>: fit %t, err %v",
|
|
task.Namespace, task.Name, node.Name, fit, err)
|
|
|
|
if !fit {
|
|
return api.NewFitErrorByReasons(task, node, reasons...)
|
|
}
|
|
|
|
if predicate.memoryPressureEnable {
|
|
// CheckNodeMemoryPressurePredicate
|
|
fit, reasons, err = predicates.CheckNodeMemoryPressurePredicate(task.Pod, nil, nodeInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
klog.V(4).Infof("CheckNodeMemoryPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v",
|
|
task.Namespace, task.Name, node.Name, fit, err)
|
|
|
|
if !fit {
|
|
return api.NewFitErrorByReasons(task, node, reasons...)
|
|
}
|
|
}
|
|
|
|
if predicate.diskPressureEnable {
|
|
// CheckNodeDiskPressurePredicate
|
|
fit, reasons, err = predicates.CheckNodeDiskPressurePredicate(task.Pod, nil, nodeInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
klog.V(4).Infof("CheckNodeDiskPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v",
|
|
task.Namespace, task.Name, node.Name, fit, err)
|
|
|
|
if !fit {
|
|
return api.NewFitErrorByReasons(task, node, reasons...)
|
|
}
|
|
}
|
|
|
|
if predicate.pidPressureEnable {
|
|
// CheckNodePIDPressurePredicate
|
|
fit, reasons, err = predicates.CheckNodePIDPressurePredicate(task.Pod, nil, nodeInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
klog.V(4).Infof("CheckNodePIDPressurePredicate predicates Task <%s/%s> on Node <%s>: fit %t, err %v",
|
|
task.Namespace, task.Name, node.Name, fit, err)
|
|
|
|
if !fit {
|
|
return api.NewFitErrorByReasons(task, node, reasons...)
|
|
}
|
|
}
|
|
|
|
var lister algorithm.PodLister
|
|
lister = pl
|
|
if !util.HaveAffinity(task.Pod) {
|
|
// pod without affinity will be only affected by pod with affinity
|
|
lister = pl.AffinityLister()
|
|
}
|
|
// Pod Affinity/Anti-Affinity Predicate
|
|
podAffinityPredicate := predicates.NewPodAffinityPredicate(ni, lister)
|
|
fit, reasons, err = podAffinityPredicate(task.Pod, nil, nodeInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
klog.V(4).Infof("Pod Affinity/Anti-Affinity predicates Task <%s/%s> on Node <%s>: fit %t, err %v",
|
|
task.Namespace, task.Name, node.Name, fit, err)
|
|
|
|
if !fit {
|
|
return api.NewFitErrorByReasons(task, node, reasons...)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (pp *predicatesPlugin) OnSessionClose(ssn *framework.Session) {}
|