Merge pull request #1850 from hanweisen/adde2etest

[E2E] add karmadactl cordon/uncordon e2e test

pkg/karmadactl/cordon.go

@@ -26,8 +26,11 @@ var (
 )
 
 const (
-	desiredCordon = iota
+	// DesiredCordon a flag indicate karmadactl.RunCordonOrUncordon cordon a cluster,
-	desiredUnCordon
+	// cordon prevent new resource scheduler to cordoned cluster.
+	DesiredCordon = iota
+	// DesiredUnCordon a flag indicate karmadactl.RunCordonOrUncordon uncordon a cluster.
+	DesiredUnCordon
 )
 
 // NewCmdCordon defines the `cordon` command that mark cluster as unschedulable.
@@ -43,7 +46,7 @@ func NewCmdCordon(karmadaConfig KarmadaConfig, parentCommand string) *cobra.Comm
 			if err := opts.Complete(args); err != nil {
 				return err
 			}
-			if err := RunCordonOrUncordon(desiredCordon, karmadaConfig, opts); err != nil {
+			if err := RunCordonOrUncordon(DesiredCordon, karmadaConfig, opts); err != nil {
 				return err
 			}
 			return nil
@@ -78,7 +81,7 @@ func NewCmdUncordon(karmadaConfig KarmadaConfig, parentCommand string) *cobra.Co
 			if err := opts.Complete(args); err != nil {
 				return err
 			}
-			if err := RunCordonOrUncordon(desiredUnCordon, karmadaConfig, opts); err != nil {
+			if err := RunCordonOrUncordon(DesiredUnCordon, karmadaConfig, opts); err != nil {
 				return err
 			}
 			return nil
@@ -139,11 +142,11 @@ func NewCordonHelper(cluster *clusterv1alpha1.Cluster) *CordonHelper {
 func (c *CordonHelper) UpdateIfRequired(desired int) bool {
 	c.desired = desired
 
-	if desired == desiredCordon && !c.hasUnschedulerTaint() {
+	if desired == DesiredCordon && !c.hasUnschedulerTaint() {
 		return true
 	}
 
-	if desired == desiredUnCordon && c.hasUnschedulerTaint() {
+	if desired == DesiredUnCordon && c.hasUnschedulerTaint() {
 		return true
 	}
 
@@ -180,11 +183,11 @@ func (c *CordonHelper) PatchOrReplace(controlPlaneClient *karmadaclientset.Clien
 		Effect: corev1.TaintEffectNoSchedule,
 	}
 
-	if c.desired == desiredCordon {
+	if c.desired == DesiredCordon {
 		c.cluster.Spec.Taints = append(c.cluster.Spec.Taints, unschedulerTaint)
 	}
 
-	if c.desired == desiredUnCordon {
+	if c.desired == DesiredUnCordon {
 		for i, n := 0, len(c.cluster.Spec.Taints); i < n; i++ {
 			if c.cluster.Spec.Taints[i].MatchTaint(&unschedulerTaint) {
 				c.cluster.Spec.Taints[i] = c.cluster.Spec.Taints[n-1]
@@ -212,7 +215,7 @@ func (c *CordonHelper) PatchOrReplace(controlPlaneClient *karmadaclientset.Clien
 // if true cordon cluster otherwise uncordon cluster.
 func RunCordonOrUncordon(desired int, karmadaConfig KarmadaConfig, opts CommandCordonOption) error {
 	cordonOrUncordon := "cordon"
-	if desired == desiredUnCordon {
+	if desired == DesiredUnCordon {
 		cordonOrUncordon = "un" + cordonOrUncordon
 	}
 
@@ -248,7 +251,7 @@ func RunCordonOrUncordon(desired int, karmadaConfig KarmadaConfig, opts CommandC
 }
 
 func alreadyStr(desired int) string {
-	if desired == desiredCordon {
+	if desired == DesiredCordon {
 		return "already cordoned"
 	}
 	return "already uncordoned"

test/e2e/karmadactl_test.go

@@ -18,6 +18,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
 	policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
@@ -409,3 +410,161 @@ var _ = framework.SerialDescribe("Karmadactl unjoin testing", ginkgo.Labels{Need
 		})
 	})
 })
+
+var _ = framework.SerialDescribe("Karmadactl cordon/uncordon testing", ginkgo.Labels{NeedCreateCluster}, func() {
+	var member1 string
+	var controlPlane string
+	var clusterName string
+	var homeDir string
+	var kubeConfigPath string
+	var clusterContext string
+	var deploymentName, deploymentNamespace string
+	var policyName, policyNamespace string
+	var deployment *appsv1.Deployment
+	var policy *policyv1alpha1.PropagationPolicy
+	var karmadaConfig karmadactl.KarmadaConfig
+
+	ginkgo.BeforeEach(func() {
+		member1 = "member1"
+		clusterName = "member-e2e-" + rand.String(3)
+		homeDir = os.Getenv("HOME")
+		kubeConfigPath = fmt.Sprintf("%s/.kube/%s.config", homeDir, clusterName)
+		controlPlane = fmt.Sprintf("%s-control-plane", clusterName)
+		clusterContext = fmt.Sprintf("kind-%s", clusterName)
+		deploymentName = deploymentNamePrefix + rand.String(RandomStrLength)
+		deploymentNamespace = testNamespace
+		policyName = deploymentName
+		policyNamespace = testNamespace
+
+		deployment = helper.NewDeployment(deploymentNamespace, deploymentName)
+		policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
+			{
+				APIVersion: deployment.APIVersion,
+				Kind:       deployment.Kind,
+				Name:       deployment.Name,
+			},
+		}, policyv1alpha1.Placement{
+			ClusterAffinity: &policyv1alpha1.ClusterAffinity{
+				ClusterNames: []string{member1, clusterName},
+			},
+			ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{
+				ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDuplicated,
+			},
+		})
+		karmadaConfig = karmadactl.NewKarmadaConfig(clientcmd.NewDefaultPathOptions())
+	})
+
+	ginkgo.BeforeEach(func() {
+		ginkgo.By(fmt.Sprintf("Creating cluster: %s", clusterName), func() {
+			err := createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext)
+			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+		})
+		ginkgo.By(fmt.Sprintf("Joinning cluster: %s", clusterName), func() {
+			karmadaConfig := karmadactl.NewKarmadaConfig(clientcmd.NewDefaultPathOptions())
+			opts := karmadactl.CommandJoinOption{
+				GlobalCommandOptions: options.GlobalCommandOptions{
+					DryRun: false,
+				},
+				ClusterNamespace:  "karmada-cluster",
+				ClusterName:       clusterName,
+				ClusterContext:    clusterContext,
+				ClusterKubeConfig: kubeConfigPath,
+			}
+			err := karmadactl.RunJoin(karmadaConfig, opts)
+			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+		})
+		// When a newly joined cluster is unready at the beginning, the scheduler will ignore it.
+		ginkgo.By(fmt.Sprintf("wait cluster %s ready", clusterName), func() {
+			framework.WaitClusterFitWith(controlPlaneClient, clusterName, func(cluster *clusterv1alpha1.Cluster) bool {
+				return meta.IsStatusConditionPresentAndEqual(cluster.Status.Conditions, clusterv1alpha1.ClusterConditionReady, metav1.ConditionTrue)
+			})
+		})
+		ginkgo.DeferCleanup(func() {
+			ginkgo.By(fmt.Sprintf("Unjoinning cluster: %s", clusterName), func() {
+				opts := karmadactl.CommandUnjoinOption{
+					GlobalCommandOptions: options.GlobalCommandOptions{
+						DryRun: false,
+					},
+					ClusterNamespace:  "karmada-cluster",
+					ClusterName:       clusterName,
+					ClusterContext:    clusterContext,
+					ClusterKubeConfig: kubeConfigPath,
+					Wait:              options.DefaultKarmadactlCommandDuration,
+				}
+				err := karmadactl.RunUnjoin(karmadaConfig, opts)
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+			})
+			ginkgo.By(fmt.Sprintf("Deleting clusters: %s", clusterName), func() {
+				err := deleteCluster(clusterName, kubeConfigPath)
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				_ = os.Remove(kubeConfigPath)
+			})
+		})
+	})
+	ginkgo.Context("cordon cluster", func() {
+		ginkgo.BeforeEach(func() {
+			opts := karmadactl.CommandCordonOption{
+				GlobalCommandOptions: options.GlobalCommandOptions{
+					DryRun: false,
+				},
+				ClusterName: clusterName,
+			}
+			err := karmadactl.RunCordonOrUncordon(karmadactl.DesiredCordon, karmadaConfig, opts)
+			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+			ginkgo.By("cluster should have a taint", func() {
+				clusterObj := &clusterv1alpha1.Cluster{}
+				err := controlPlaneClient.Get(context.TODO(), client.ObjectKey{Name: clusterName}, clusterObj)
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				taints := clusterObj.Spec.Taints
+				var unschedulable corev1.Taint
+				for index := range taints {
+					if taints[index].Key == clusterv1alpha1.TaintClusterUnscheduler {
+						unschedulable = taints[index]
+						break
+					}
+				}
+				gomega.Expect(unschedulable).ShouldNot(gomega.BeNil())
+				gomega.Expect(unschedulable.Effect).Should(gomega.Equal(corev1.TaintEffectNoSchedule))
+			})
+		})
+
+		ginkgo.It(fmt.Sprintf("deploy deployment should not schedule to cordon cluster %s", clusterName), func() {
+			framework.CreatePropagationPolicy(karmadaClient, policy)
+			framework.CreateDeployment(kubeClient, deployment)
+			ginkgo.DeferCleanup(func() {
+				framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
+				framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name)
+			})
+			targetClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+			gomega.Expect(targetClusters).ShouldNot(gomega.ContainElement(clusterName))
+		})
+
+		ginkgo.It("uncordon cluster", func() {
+			opts := karmadactl.CommandCordonOption{
+				GlobalCommandOptions: options.GlobalCommandOptions{
+					DryRun: false,
+				},
+				ClusterName: clusterName,
+			}
+			err := karmadactl.RunCordonOrUncordon(karmadactl.DesiredUnCordon, karmadaConfig, opts)
+			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+			ginkgo.By(fmt.Sprintf("cluster %s taint will be removed", clusterName), func() {
+				clusterObj := &clusterv1alpha1.Cluster{}
+				err := controlPlaneClient.Get(context.TODO(), client.ObjectKey{Name: clusterName}, clusterObj)
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				gomega.Expect(clusterObj.Spec.Taints).Should(gomega.BeEmpty())
+			})
+
+			framework.CreatePropagationPolicy(karmadaClient, policy)
+			framework.CreateDeployment(kubeClient, deployment)
+			ginkgo.DeferCleanup(func() {
+				framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
+				framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name)
+			})
+			ginkgo.By(fmt.Sprintf("deployment will schedule to cluster %s", clusterName), func() {
+				targetClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+				gomega.Expect(targetClusters).Should(gomega.ContainElement(clusterName))
+			})
+		})
+	})
+})