diff --git a/pkg/karmadactl/cordon.go b/pkg/karmadactl/cordon.go index b8360f4cb..de9fd4894 100644 --- a/pkg/karmadactl/cordon.go +++ b/pkg/karmadactl/cordon.go @@ -26,8 +26,11 @@ var ( ) const ( - desiredCordon = iota - desiredUnCordon + // DesiredCordon a flag indicate karmadactl.RunCordonOrUncordon cordon a cluster, + // cordon prevent new resource scheduler to cordoned cluster. + DesiredCordon = iota + // DesiredUnCordon a flag indicate karmadactl.RunCordonOrUncordon uncordon a cluster. + DesiredUnCordon ) // NewCmdCordon defines the `cordon` command that mark cluster as unschedulable. @@ -43,7 +46,7 @@ func NewCmdCordon(karmadaConfig KarmadaConfig, parentCommand string) *cobra.Comm if err := opts.Complete(args); err != nil { return err } - if err := RunCordonOrUncordon(desiredCordon, karmadaConfig, opts); err != nil { + if err := RunCordonOrUncordon(DesiredCordon, karmadaConfig, opts); err != nil { return err } return nil @@ -78,7 +81,7 @@ func NewCmdUncordon(karmadaConfig KarmadaConfig, parentCommand string) *cobra.Co if err := opts.Complete(args); err != nil { return err } - if err := RunCordonOrUncordon(desiredUnCordon, karmadaConfig, opts); err != nil { + if err := RunCordonOrUncordon(DesiredUnCordon, karmadaConfig, opts); err != nil { return err } return nil @@ -139,11 +142,11 @@ func NewCordonHelper(cluster *clusterv1alpha1.Cluster) *CordonHelper { func (c *CordonHelper) UpdateIfRequired(desired int) bool { c.desired = desired - if desired == desiredCordon && !c.hasUnschedulerTaint() { + if desired == DesiredCordon && !c.hasUnschedulerTaint() { return true } - if desired == desiredUnCordon && c.hasUnschedulerTaint() { + if desired == DesiredUnCordon && c.hasUnschedulerTaint() { return true } @@ -180,11 +183,11 @@ func (c *CordonHelper) PatchOrReplace(controlPlaneClient *karmadaclientset.Clien Effect: corev1.TaintEffectNoSchedule, } - if c.desired == desiredCordon { + if c.desired == DesiredCordon { c.cluster.Spec.Taints = append(c.cluster.Spec.Taints, unschedulerTaint) } - if c.desired == desiredUnCordon { + if c.desired == DesiredUnCordon { for i, n := 0, len(c.cluster.Spec.Taints); i < n; i++ { if c.cluster.Spec.Taints[i].MatchTaint(&unschedulerTaint) { c.cluster.Spec.Taints[i] = c.cluster.Spec.Taints[n-1] @@ -212,7 +215,7 @@ func (c *CordonHelper) PatchOrReplace(controlPlaneClient *karmadaclientset.Clien // if true cordon cluster otherwise uncordon cluster. func RunCordonOrUncordon(desired int, karmadaConfig KarmadaConfig, opts CommandCordonOption) error { cordonOrUncordon := "cordon" - if desired == desiredUnCordon { + if desired == DesiredUnCordon { cordonOrUncordon = "un" + cordonOrUncordon } @@ -248,7 +251,7 @@ func RunCordonOrUncordon(desired int, karmadaConfig KarmadaConfig, opts CommandC } func alreadyStr(desired int) string { - if desired == desiredCordon { + if desired == DesiredCordon { return "already cordoned" } return "already uncordoned" diff --git a/test/e2e/karmadactl_test.go b/test/e2e/karmadactl_test.go index 38b0fe00f..c16367342 100644 --- a/test/e2e/karmadactl_test.go +++ b/test/e2e/karmadactl_test.go @@ -18,6 +18,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" @@ -409,3 +410,161 @@ var _ = framework.SerialDescribe("Karmadactl unjoin testing", ginkgo.Labels{Need }) }) }) + +var _ = framework.SerialDescribe("Karmadactl cordon/uncordon testing", ginkgo.Labels{NeedCreateCluster}, func() { + var member1 string + var controlPlane string + var clusterName string + var homeDir string + var kubeConfigPath string + var clusterContext string + var deploymentName, deploymentNamespace string + var policyName, policyNamespace string + var deployment *appsv1.Deployment + var policy *policyv1alpha1.PropagationPolicy + var karmadaConfig karmadactl.KarmadaConfig + + ginkgo.BeforeEach(func() { + member1 = "member1" + clusterName = "member-e2e-" + rand.String(3) + homeDir = os.Getenv("HOME") + kubeConfigPath = fmt.Sprintf("%s/.kube/%s.config", homeDir, clusterName) + controlPlane = fmt.Sprintf("%s-control-plane", clusterName) + clusterContext = fmt.Sprintf("kind-%s", clusterName) + deploymentName = deploymentNamePrefix + rand.String(RandomStrLength) + deploymentNamespace = testNamespace + policyName = deploymentName + policyNamespace = testNamespace + + deployment = helper.NewDeployment(deploymentNamespace, deploymentName) + policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deployment.Name, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{member1, clusterName}, + }, + ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{ + ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDuplicated, + }, + }) + karmadaConfig = karmadactl.NewKarmadaConfig(clientcmd.NewDefaultPathOptions()) + }) + + ginkgo.BeforeEach(func() { + ginkgo.By(fmt.Sprintf("Creating cluster: %s", clusterName), func() { + err := createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + ginkgo.By(fmt.Sprintf("Joinning cluster: %s", clusterName), func() { + karmadaConfig := karmadactl.NewKarmadaConfig(clientcmd.NewDefaultPathOptions()) + opts := karmadactl.CommandJoinOption{ + GlobalCommandOptions: options.GlobalCommandOptions{ + DryRun: false, + }, + ClusterNamespace: "karmada-cluster", + ClusterName: clusterName, + ClusterContext: clusterContext, + ClusterKubeConfig: kubeConfigPath, + } + err := karmadactl.RunJoin(karmadaConfig, opts) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + // When a newly joined cluster is unready at the beginning, the scheduler will ignore it. + ginkgo.By(fmt.Sprintf("wait cluster %s ready", clusterName), func() { + framework.WaitClusterFitWith(controlPlaneClient, clusterName, func(cluster *clusterv1alpha1.Cluster) bool { + return meta.IsStatusConditionPresentAndEqual(cluster.Status.Conditions, clusterv1alpha1.ClusterConditionReady, metav1.ConditionTrue) + }) + }) + ginkgo.DeferCleanup(func() { + ginkgo.By(fmt.Sprintf("Unjoinning cluster: %s", clusterName), func() { + opts := karmadactl.CommandUnjoinOption{ + GlobalCommandOptions: options.GlobalCommandOptions{ + DryRun: false, + }, + ClusterNamespace: "karmada-cluster", + ClusterName: clusterName, + ClusterContext: clusterContext, + ClusterKubeConfig: kubeConfigPath, + Wait: options.DefaultKarmadactlCommandDuration, + } + err := karmadactl.RunUnjoin(karmadaConfig, opts) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + ginkgo.By(fmt.Sprintf("Deleting clusters: %s", clusterName), func() { + err := deleteCluster(clusterName, kubeConfigPath) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + _ = os.Remove(kubeConfigPath) + }) + }) + }) + ginkgo.Context("cordon cluster", func() { + ginkgo.BeforeEach(func() { + opts := karmadactl.CommandCordonOption{ + GlobalCommandOptions: options.GlobalCommandOptions{ + DryRun: false, + }, + ClusterName: clusterName, + } + err := karmadactl.RunCordonOrUncordon(karmadactl.DesiredCordon, karmadaConfig, opts) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + ginkgo.By("cluster should have a taint", func() { + clusterObj := &clusterv1alpha1.Cluster{} + err := controlPlaneClient.Get(context.TODO(), client.ObjectKey{Name: clusterName}, clusterObj) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + taints := clusterObj.Spec.Taints + var unschedulable corev1.Taint + for index := range taints { + if taints[index].Key == clusterv1alpha1.TaintClusterUnscheduler { + unschedulable = taints[index] + break + } + } + gomega.Expect(unschedulable).ShouldNot(gomega.BeNil()) + gomega.Expect(unschedulable.Effect).Should(gomega.Equal(corev1.TaintEffectNoSchedule)) + }) + }) + + ginkgo.It(fmt.Sprintf("deploy deployment should not schedule to cordon cluster %s", clusterName), func() { + framework.CreatePropagationPolicy(karmadaClient, policy) + framework.CreateDeployment(kubeClient, deployment) + ginkgo.DeferCleanup(func() { + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name) + }) + targetClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment) + gomega.Expect(targetClusters).ShouldNot(gomega.ContainElement(clusterName)) + }) + + ginkgo.It("uncordon cluster", func() { + opts := karmadactl.CommandCordonOption{ + GlobalCommandOptions: options.GlobalCommandOptions{ + DryRun: false, + }, + ClusterName: clusterName, + } + err := karmadactl.RunCordonOrUncordon(karmadactl.DesiredUnCordon, karmadaConfig, opts) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("cluster %s taint will be removed", clusterName), func() { + clusterObj := &clusterv1alpha1.Cluster{} + err := controlPlaneClient.Get(context.TODO(), client.ObjectKey{Name: clusterName}, clusterObj) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(clusterObj.Spec.Taints).Should(gomega.BeEmpty()) + }) + + framework.CreatePropagationPolicy(karmadaClient, policy) + framework.CreateDeployment(kubeClient, deployment) + ginkgo.DeferCleanup(func() { + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name) + }) + ginkgo.By(fmt.Sprintf("deployment will schedule to cluster %s", clusterName), func() { + targetClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment) + gomega.Expect(targetClusters).Should(gomega.ContainElement(clusterName)) + }) + }) + }) +})