karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add a clusterEvicted plugin 

Signed-off-by: Poor12 <shentiecheng@huawei.com>
This commit is contained in:
Poor12 2023-04-26 15:09:28 +08:00
parent 0f84dd76a9
commit 80eab7bcb3
5 changed files with 246 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
pkg/scheduler/framework/plugins/clustereviction/cluster_eviction.go Normal file
View File

@ -0,0 +1,42 @@
package clustereviction
import (
"context"
"k8s.io/klog/v2"
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
"github.com/karmada-io/karmada/pkg/scheduler/framework"
"github.com/karmada-io/karmada/pkg/util/helper"
)
const (
// Name is the name of the plugin used in the plugin registry and configurations.
Name = "ClusterEviction"
)
// ClusterEviction is a plugin that checks if the target cluster is in the GracefulEvictionTasks which means it is in the process of eviction.
type ClusterEviction struct{}
var _ framework.FilterPlugin = &ClusterEviction{}
// New instantiates the ClusterEviction plugin.
func New() (framework.Plugin, error) {
return &ClusterEviction{}, nil
}
// Name returns the plugin name.
func (p *ClusterEviction) Name() string {
return Name
}
// Filter checks if the target cluster is in the GracefulEvictionTasks which means it is in the process of eviction.
func (p *ClusterEviction) Filter(_ context.Context, bindingSpec *workv1alpha2.ResourceBindingSpec, _ *workv1alpha2.ResourceBindingStatus, cluster *clusterv1alpha1.Cluster) *framework.Result {
if helper.ClusterInGracefulEvictionTasks(bindingSpec.GracefulEvictionTasks, cluster.Name) {
klog.V(2).Infof("Cluster(%s) is in the process of eviction.", cluster.Name)
return framework.NewResult(framework.Unschedulable, "cluster(s) is in the process of eviction")
}
return framework.NewResult(framework.Success)
}

2
pkg/scheduler/framework/plugins/registry.go
View File

@ -3,6 +3,7 @@ package plugins
import (
"github.com/karmada-io/karmada/pkg/scheduler/framework/plugins/apienablement"
"github.com/karmada-io/karmada/pkg/scheduler/framework/plugins/clusteraffinity"
"github.com/karmada-io/karmada/pkg/scheduler/framework/plugins/clustereviction"
"github.com/karmada-io/karmada/pkg/scheduler/framework/plugins/clusterlocality"
"github.com/karmada-io/karmada/pkg/scheduler/framework/plugins/spreadconstraint"
"github.com/karmada-io/karmada/pkg/scheduler/framework/plugins/tainttoleration"
@ -17,5 +18,6 @@ func NewInTreeRegistry() runtime.Registry {
clusteraffinity.Name: clusteraffinity.New,
spreadconstraint.Name: spreadconstraint.New,
clusterlocality.Name: clusterlocality.New,
clustereviction.Name: clustereviction.New,
}
}

12
pkg/util/helper/cluster.go
View File

@ -2,6 +2,7 @@ package helper
import (
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
)
// IsAPIEnabled checks if target API (or CRD) referencing by groupVersion and kind has been installed.
@ -21,3 +22,14 @@ func IsAPIEnabled(APIEnablements []clusterv1alpha1.APIEnablement, groupVersion s
return false
}
// ClusterInGracefulEvictionTasks checks if the target cluster is in the GracefulEvictionTasks which means it is in the process of eviction.
func ClusterInGracefulEvictionTasks(gracefulEvictionTasks []workv1alpha2.GracefulEvictionTask, clusterName string) bool {
for _, task := range gracefulEvictionTasks {
if task.FromCluster == clusterName {
return true
}
}
return false
}

46
pkg/util/helper/cluster_test.go
View File

@ -4,6 +4,7 @@ import (
"testing"
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
)
func TestIsAPIEnabled(t *testing.T) {
@ -72,3 +73,48 @@ func TestIsAPIEnabled(t *testing.T) {
})
}
}
func TestClusterInGracefulEvictionTasks(t *testing.T) {
gracefulEvictionTasks := []workv1alpha2.GracefulEvictionTask{
{
FromCluster: ClusterMember1,
Producer: workv1alpha2.EvictionProducerTaintManager,
Reason: workv1alpha2.EvictionReasonTaintUntolerated,
},
{
FromCluster: ClusterMember2,
Producer: workv1alpha2.EvictionProducerTaintManager,
Reason: workv1alpha2.EvictionReasonTaintUntolerated,
},
}
tests := []struct {
name string
gracefulEvictionTasks []workv1alpha2.GracefulEvictionTask
targetCluster string
expect bool
}{
{
name: "targetCluster is in the process of eviction",
gracefulEvictionTasks: gracefulEvictionTasks,
targetCluster: ClusterMember1,
expect: true,
},
{
name: "targetCluster is not in the process of eviction",
gracefulEvictionTasks: gracefulEvictionTasks,
targetCluster: ClusterMember3,
expect: false,
},
}
for _, test := range tests {
tc := test
t.Run(tc.name, func(t *testing.T) {
result := ClusterInGracefulEvictionTasks(tc.gracefulEvictionTasks, tc.targetCluster)
if result != tc.expect {
t.Errorf("expected: %v, but got: %v", tc.expect, result)
}
})
}
}

144
test/e2e/failover_test.go
View File

@ -13,6 +13,7 @@ import (
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
@ -141,6 +142,106 @@ var _ = framework.SerialDescribe("failover testing", func() {
})
})
})
ginkgo.Context("Taint cluster testing", func() {
var policyNamespace, policyName string
var deploymentNamespace, deploymentName string
var deployment *appsv1.Deployment
var taint corev1.Taint
var maxGroups, minGroups, numOfFailedClusters int
var policy *policyv1alpha1.PropagationPolicy
maxGroups = 1
minGroups = 1
numOfFailedClusters = 1
ginkgo.BeforeEach(func() {
policyNamespace = testNamespace
policyName = deploymentNamePrefix + rand.String(RandomStrLength)
deploymentNamespace = testNamespace
deploymentName = policyName
deployment = testhelper.NewDeployment(deploymentNamespace, deploymentName)
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
{
APIVersion: deployment.APIVersion,
Kind: deployment.Kind,
Name: deployment.Name,
},
}, policyv1alpha1.Placement{
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
ClusterNames: framework.ClusterNames(),
},
ClusterTolerations: []corev1.Toleration{
{
Key: "fail-test",
Effect: corev1.TaintEffectNoExecute,
Operator: corev1.TolerationOpExists,
TolerationSeconds: pointer.Int64(3),
},
},
SpreadConstraints: []policyv1alpha1.SpreadConstraint{
{
SpreadByField: policyv1alpha1.SpreadByFieldCluster,
MaxGroups: maxGroups,
MinGroups: minGroups,
},
},
})
taint = corev1.Taint{
Key: "fail-test",
Effect: corev1.TaintEffectNoExecute,
}
})
ginkgo.BeforeEach(func() {
framework.CreatePropagationPolicy(karmadaClient, policy)
framework.CreateDeployment(kubeClient, deployment)
ginkgo.DeferCleanup(func() {
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name)
})
})
ginkgo.It("taint cluster", func() {
var disabledClusters []string
targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
ginkgo.By("taint one cluster", func() {
temp := numOfFailedClusters
for _, targetClusterName := range targetClusterNames {
if temp > 0 {
klog.Infof("Taint one cluster(%s).", targetClusterName)
err := taintCluster(controlPlaneClient, targetClusterName, taint)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
disabledClusters = append(disabledClusters, targetClusterName)
temp--
}
}
})
ginkgo.By("check whether deployment of failed cluster is rescheduled to other available cluster", func() {
gomega.Eventually(func() int {
targetClusterNames = framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
for _, targetClusterName := range targetClusterNames {
// the target cluster should be overwritten to another available cluster
if !testhelper.IsExclude(targetClusterName, disabledClusters) {
return 0
}
}
return len(targetClusterNames)
}, pollTimeout, pollInterval).Should(gomega.Equal(minGroups))
})
ginkgo.By("recover not ready cluster", func() {
for _, disabledCluster := range disabledClusters {
fmt.Printf("cluster %s is waiting for recovering\n", disabledCluster)
err := recoverTaintedCluster(controlPlaneClient, disabledCluster, taint)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
}
})
})
})
})
// disableCluster will set wrong API endpoint of current cluster
@ -167,6 +268,49 @@ func disableCluster(c client.Client, clusterName string) error {
return err
}
// taintCluster will taint cluster
func taintCluster(c client.Client, clusterName string, taint corev1.Taint) error {
err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) {
clusterObj := &clusterv1alpha1.Cluster{}
if err := c.Get(context.TODO(), client.ObjectKey{Name: clusterName}, clusterObj); err != nil {
if apierrors.IsConflict(err) {
return false, nil
}
return false, err
}
clusterObj.Spec.Taints = append(clusterObj.Spec.Taints, taint)
if err := c.Update(context.TODO(), clusterObj); err != nil {
if apierrors.IsConflict(err) {
return false, nil
}
return false, err
}
return true, nil
})
return err
}
// recoverTaintedCluster will recover the taint of the disabled cluster
func recoverTaintedCluster(c client.Client, clusterName string, taint corev1.Taint) error {
err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) {
clusterObj := &clusterv1alpha1.Cluster{}
if err := c.Get(context.TODO(), client.ObjectKey{Name: clusterName}, clusterObj); err != nil {
if apierrors.IsConflict(err) {
return false, nil
}
return false, err
}
if err := helper.UpdateClusterControllerTaint(context.TODO(), c, nil, []*corev1.Taint{&taint}, clusterObj); err != nil {
if apierrors.IsConflict(err) {
return false, nil
}
return false, err
}
return true, nil
})
return err
}
// recoverCluster will recover API endpoint of the disable cluster
func recoverCluster(c client.Client, clusterName string, originalAPIEndpoint string) error {
err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) {