feat: cluster-level resource scheduling suspend and resume capabilities (#5937)

* RB suspension: API change

Signed-off-by: Monokaix <changxuzheng@huawei.com>

* Feat: scheduler supports binging suspension

Signed-off-by: Monokaix <changxuzheng@huawei.com>

---------

Signed-off-by: Monokaix <changxuzheng@huawei.com>

api/openapi-spec/swagger.json

@@ -20435,7 +20435,7 @@
       }
     },
     "com.github.karmada-io.karmada.pkg.apis.work.v1alpha2.Suspension": {
-      "description": "Suspension defines the policy for suspending of propagation.",
+      "description": "Suspension defines the policy for suspending dispatching and scheduling.",
       "type": "object",
       "properties": {
         "dispatching": {
@@ -20445,6 +20445,10 @@
         "dispatchingOnClusters": {
           "description": "DispatchingOnClusters declares a list of clusters to which the dispatching should be suspended. Note: Can not co-exist with Dispatching which is used to suspend all.",
           "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.policy.v1alpha1.SuspendClusters"
+        },
+        "scheduling": {
+          "description": "Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not process resource binding when the value is true and resume scheduling when it's false or nil. This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling manage resource allocation, prioritize critical workloads, etc. It is expected that third-party systems use an admission webhook to suspend scheduling at the time of ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may lead to ineffective suspension.",
+          "type": "boolean"
         }
       }
     },

charts/karmada/_crds/bases/work/work.karmada.io_clusterresourcebindings.yaml

@@ -1281,6 +1281,16 @@ spec:
                           type: string
                         type: array
                     type: object
+                  scheduling:
+                    description: |-
+                      Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not
+                      process resource binding when the value is true and resume scheduling when it's false or nil.
+                      This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling
+                      manage resource allocation, prioritize critical workloads, etc.
+                      It is expected that third-party systems use an admission webhook to suspend scheduling at the time of
+                      ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may
+                      lead to ineffective suspension.
+                    type: boolean
                 type: object
             required:
             - resource

charts/karmada/_crds/bases/work/work.karmada.io_resourcebindings.yaml

@@ -1281,6 +1281,16 @@ spec:
                           type: string
                         type: array
                     type: object
+                  scheduling:
+                    description: |-
+                      Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not
+                      process resource binding when the value is true and resume scheduling when it's false or nil.
+                      This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling
+                      manage resource allocation, prioritize critical workloads, etc.
+                      It is expected that third-party systems use an admission webhook to suspend scheduling at the time of
+                      ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may
+                      lead to ineffective suspension.
+                    type: boolean
                 type: object
             required:
             - resource

pkg/apis/work/v1alpha2/binding_types.go

@@ -322,9 +322,19 @@ type BindingSnapshot struct {
 	Clusters []TargetCluster `json:"clusters,omitempty"`
 }
 
-// Suspension defines the policy for suspending of propagation.
+// Suspension defines the policy for suspending dispatching and scheduling.
 type Suspension struct {
 	policyv1alpha1.Suspension `json:",inline"`
+
+	// Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not
+	// process resource binding when the value is true and resume scheduling when it's false or nil.
+	// This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling
+	// manage resource allocation, prioritize critical workloads, etc.
+	// It is expected that third-party systems use an admission webhook to suspend scheduling at the time of
+	// ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may
+	// lead to ineffective suspension.
+	// +optional
+	Scheduling *bool `json:"scheduling,omitempty"`
 }
 
 // ResourceBindingStatus represents the overall status of the strategy as well as the referenced resources.

pkg/apis/work/v1alpha2/zz_generated.deepcopy.go

@@ -421,6 +421,11 @@ func (in *ResourceBindingStatus) DeepCopy() *ResourceBindingStatus {
 func (in *Suspension) DeepCopyInto(out *Suspension) {
 	*out = *in
 	in.Suspension.DeepCopyInto(&out.Suspension)
+	if in.Scheduling != nil {
+		in, out := &in.Scheduling, &out.Scheduling
+		*out = new(bool)
+		**out = **in
+	}
 	return
 }
 

pkg/generated/openapi/zz_generated.openapi.go

@@ -7461,7 +7461,7 @@ func schema_pkg_apis_work_v1alpha2_Suspension(ref common.ReferenceCallback) comm
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{
 			SchemaProps: spec.SchemaProps{
-				Description: "Suspension defines the policy for suspending of propagation.",
+				Description: "Suspension defines the policy for suspending dispatching and scheduling.",
 				Type:        []string{"object"},
 				Properties: map[string]spec.Schema{
 					"dispatching": {
@@ -7477,6 +7477,13 @@ func schema_pkg_apis_work_v1alpha2_Suspension(ref common.ReferenceCallback) comm
 							Ref:         ref("github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1.SuspendClusters"),
 						},
 					},
+					"scheduling": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Scheduling controls whether scheduling should be suspended, the scheduler will pause scheduling and not process resource binding when the value is true and resume scheduling when it's false or nil. This is designed for third-party systems to temporarily pause the scheduling of applications, which enabling manage resource allocation, prioritize critical workloads, etc. It is expected that third-party systems use an admission webhook to suspend scheduling at the time of ResourceBinding creation. Once a ResourceBinding has been scheduled, it cannot be paused afterward, as it may lead to ineffective suspension.",
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
 				},
 			},
 		},

pkg/scheduler/event_handler.go

@@ -101,10 +101,16 @@ func (s *Scheduler) resourceBindingEventFilter(obj interface{}) bool {
 		if !schedulerNameFilter(s.schedulerName, t.Spec.SchedulerName) {
 			return false
 		}
+		if util.IsBindingSuspendScheduling(t) {
+			return false
+		}
 	case *workv1alpha2.ClusterResourceBinding:
 		if !schedulerNameFilter(s.schedulerName, t.Spec.SchedulerName) {
 			return false
 		}
+		if util.IsClusterBindingSuspendScheduling(t) {
+			return false
+		}
 	}
 
 	return util.GetLabelValue(accessor.GetLabels(), policyv1alpha1.PropagationPolicyPermanentIDLabel) != "" ||

pkg/scheduler/event_handler_test.go

@@ -24,6 +24,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/utils/ptr"
 
 	clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
 	policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
@@ -40,13 +41,13 @@ func TestResourceBindingEventFilter(t *testing.T) {
 		{
 			name:           "ResourceBinding: Matching scheduler name, no labels",
 			schedulerName:  "test-scheduler",
-			obj:            createResourceBinding("test-rb", "test-scheduler", nil),
+			obj:            createResourceBinding("test-rb", "test-scheduler", nil, nil),
 			expectedResult: false,
 		},
 		{
 			name:           "ResourceBinding: Non-matching scheduler name",
 			schedulerName:  "test-scheduler",
-			obj:            createResourceBinding("test-rb", "other-scheduler", nil),
+			obj:            createResourceBinding("test-rb", "other-scheduler", nil, nil),
 			expectedResult: false,
 		},
 		{
@@ -54,7 +55,7 @@ func TestResourceBindingEventFilter(t *testing.T) {
 			schedulerName: "test-scheduler",
 			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
 				policyv1alpha1.PropagationPolicyPermanentIDLabel: "test-id",
-			}),
+			}, nil),
 			expectedResult: true,
 		},
 		{
@@ -62,7 +63,7 @@ func TestResourceBindingEventFilter(t *testing.T) {
 			schedulerName: "test-scheduler",
 			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
 				policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: "test-id",
-			}),
+			}, nil),
 			expectedResult: true,
 		},
 		{
@@ -70,7 +71,7 @@ func TestResourceBindingEventFilter(t *testing.T) {
 			schedulerName: "test-scheduler",
 			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
 				workv1alpha2.BindingManagedByLabel: "test-manager",
-			}),
+			}, nil),
 			expectedResult: true,
 		},
 		{
@@ -78,19 +79,19 @@ func TestResourceBindingEventFilter(t *testing.T) {
 			schedulerName: "test-scheduler",
 			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
 				policyv1alpha1.PropagationPolicyPermanentIDLabel: "",
-			}),
+			}, nil),
 			expectedResult: false,
 		},
 		{
 			name:           "ClusterResourceBinding: Matching scheduler name, no labels",
 			schedulerName:  "test-scheduler",
-			obj:            createClusterResourceBinding("test-crb", "test-scheduler", nil),
+			obj:            createClusterResourceBinding("test-crb", "test-scheduler", nil, nil),
 			expectedResult: false,
 		},
 		{
 			name:           "ClusterResourceBinding: Non-matching scheduler name",
 			schedulerName:  "test-scheduler",
-			obj:            createClusterResourceBinding("test-crb", "other-scheduler", nil),
+			obj:            createClusterResourceBinding("test-crb", "other-scheduler", nil, nil),
 			expectedResult: false,
 		},
 		{
@@ -98,7 +99,7 @@ func TestResourceBindingEventFilter(t *testing.T) {
 			schedulerName: "test-scheduler",
 			obj: createClusterResourceBinding("test-crb", "test-scheduler", map[string]string{
 				policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: "test-id",
-			}),
+			}, nil),
 			expectedResult: true,
 		},
 		{
@@ -113,6 +114,22 @@ func TestResourceBindingEventFilter(t *testing.T) {
 			obj:            "not-a-valid-object",
 			expectedResult: false,
 		},
+		{
+			name:          "ResourceBinding suspended",
+			schedulerName: "test-scheduler",
+			obj: createResourceBinding("test-rb", "test-scheduler", map[string]string{
+				workv1alpha2.BindingManagedByLabel: "test-manager",
+			}, &workv1alpha2.Suspension{Scheduling: ptr.To(true)}),
+			expectedResult: false,
+		},
+		{
+			name:          "ClusterResourceBinding suspended",
+			schedulerName: "test-scheduler",
+			obj: createClusterResourceBinding("test-crb", "test-scheduler", map[string]string{
+				policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel: "test-id",
+			}, &workv1alpha2.Suspension{Scheduling: ptr.To(true)}),
+			expectedResult: false,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -404,7 +421,7 @@ func createCluster(name string, generation int64, labels map[string]string) *clu
 	}
 }
 
-func createResourceBinding(name, schedulerName string, labels map[string]string) *workv1alpha2.ResourceBinding {
+func createResourceBinding(name, schedulerName string, labels map[string]string, suspension *workv1alpha2.Suspension) *workv1alpha2.ResourceBinding {
 	return &workv1alpha2.ResourceBinding{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   name,
@@ -412,11 +429,12 @@ func createResourceBinding(name, schedulerName string, labels map[string]string)
 		},
 		Spec: workv1alpha2.ResourceBindingSpec{
 			SchedulerName: schedulerName,
+			Suspension:    suspension,
 		},
 	}
 }
 
-func createClusterResourceBinding(name, schedulerName string, labels map[string]string) *workv1alpha2.ClusterResourceBinding {
+func createClusterResourceBinding(name, schedulerName string, labels map[string]string, suspension *workv1alpha2.Suspension) *workv1alpha2.ClusterResourceBinding {
 	return &workv1alpha2.ClusterResourceBinding{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   name,
@@ -424,6 +442,7 @@ func createClusterResourceBinding(name, schedulerName string, labels map[string]
 		},
 		Spec: workv1alpha2.ResourceBindingSpec{
 			SchedulerName: schedulerName,
+			Suspension:    suspension,
 		},
 	}
 }

pkg/util/binding.go

@@ -110,3 +110,19 @@ func RescheduleRequired(rescheduleTriggeredAt, lastScheduledTime *metav1.Time) b
 	}
 	return rescheduleTriggeredAt.After(lastScheduledTime.Time)
 }
+
+// IsBindingSuspendScheduling tells whether resource binding is scheduling suspended.
+func IsBindingSuspendScheduling(rb *workv1alpha2.ResourceBinding) bool {
+	if rb == nil || rb.Spec.Suspension == nil || rb.Spec.Suspension.Scheduling == nil {
+		return false
+	}
+	return *rb.Spec.Suspension.Scheduling
+}
+
+// IsClusterBindingSuspendScheduling tells whether cluster resource binding is scheduling suspended.
+func IsClusterBindingSuspendScheduling(crb *workv1alpha2.ClusterResourceBinding) bool {
+	if crb == nil || crb.Spec.Suspension == nil || crb.Spec.Suspension.Scheduling == nil {
+		return false
+	}
+	return *crb.Spec.Suspension.Scheduling
+}

pkg/util/binding_test.go

@@ -21,8 +21,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/utils/ptr"
 
 	policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
 	workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
@@ -419,3 +421,115 @@ func TestRescheduleRequired(t *testing.T) {
 		})
 	}
 }
+
+func TestIsBindingSuspendScheduling(t *testing.T) {
+	tests := []struct {
+		name     string
+		rb       *workv1alpha2.ResourceBinding
+		expected bool
+	}{
+		{
+			name:     "rb is nil",
+			rb:       nil,
+			expected: false,
+		},
+		{
+			name:     "rb.Spec.Suspension is nil",
+			rb:       &workv1alpha2.ResourceBinding{},
+			expected: false,
+		},
+		{
+			name: "rb.Spec.Suspension.Scheduling is nil",
+			rb: &workv1alpha2.ResourceBinding{
+				Spec: workv1alpha2.ResourceBindingSpec{
+					Suspension: &workv1alpha2.Suspension{},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "rb.Spec.Suspension.Scheduling is false",
+			rb: &workv1alpha2.ResourceBinding{
+				Spec: workv1alpha2.ResourceBindingSpec{
+					Suspension: &workv1alpha2.Suspension{
+						Scheduling: ptr.To(false),
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "rb.Spec.Suspension.Scheduling is true",
+			rb: &workv1alpha2.ResourceBinding{
+				Spec: workv1alpha2.ResourceBindingSpec{
+					Suspension: &workv1alpha2.Suspension{
+						Scheduling: ptr.To(true),
+					},
+				},
+			},
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, IsBindingSuspendScheduling(tt.rb))
+		})
+	}
+}
+
+func TestIsClusterBindingSuspendScheduling(t *testing.T) {
+	tests := []struct {
+		name     string
+		crb      *workv1alpha2.ClusterResourceBinding
+		expected bool
+	}{
+		{
+			name:     "crb is nil",
+			crb:      nil,
+			expected: false,
+		},
+		{
+			name:     "crb.Spec.Suspension is nil",
+			crb:      &workv1alpha2.ClusterResourceBinding{},
+			expected: false,
+		},
+		{
+			name: "crb.Spec.Suspension.Scheduling is nil",
+			crb: &workv1alpha2.ClusterResourceBinding{
+				Spec: workv1alpha2.ResourceBindingSpec{
+					Suspension: &workv1alpha2.Suspension{},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "crb.Spec.Suspension.Scheduling is false",
+			crb: &workv1alpha2.ClusterResourceBinding{
+				Spec: workv1alpha2.ResourceBindingSpec{
+					Suspension: &workv1alpha2.Suspension{
+						Scheduling: ptr.To(false),
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "crb.Spec.Suspension.Scheduling is true",
+			crb: &workv1alpha2.ClusterResourceBinding{
+				Spec: workv1alpha2.ResourceBindingSpec{
+					Suspension: &workv1alpha2.Suspension{
+						Scheduling: ptr.To(true),
+					},
+				},
+			},
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, IsClusterBindingSuspendScheduling(tt.crb))
+		})
+	}
+}