Merge pull request #1850 from hanweisen/adde2etest
[E2E] add karmadactl cordon/uncordon e2e test
This commit is contained in:
karmada-bot
GitHub
commit
0abb792bbf
|
|
@ -26,8 +26,11 @@ var (
|
|||
)
|
||||
|
||||
const (
|
||||
desiredCordon = iota
|
||||
desiredUnCordon
|
||||
// DesiredCordon a flag indicate karmadactl.RunCordonOrUncordon cordon a cluster,
|
||||
// cordon prevent new resource scheduler to cordoned cluster.
|
||||
DesiredCordon = iota
|
||||
// DesiredUnCordon a flag indicate karmadactl.RunCordonOrUncordon uncordon a cluster.
|
||||
DesiredUnCordon
|
||||
)
|
||||
|
||||
// NewCmdCordon defines the `cordon` command that mark cluster as unschedulable.
|
||||
|
|
@ -43,7 +46,7 @@ func NewCmdCordon(karmadaConfig KarmadaConfig, parentCommand string) *cobra.Comm
|
|||
if err := opts.Complete(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := RunCordonOrUncordon(desiredCordon, karmadaConfig, opts); err != nil {
|
||||
if err := RunCordonOrUncordon(DesiredCordon, karmadaConfig, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
|
@ -78,7 +81,7 @@ func NewCmdUncordon(karmadaConfig KarmadaConfig, parentCommand string) *cobra.Co
|
|||
if err := opts.Complete(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := RunCordonOrUncordon(desiredUnCordon, karmadaConfig, opts); err != nil {
|
||||
if err := RunCordonOrUncordon(DesiredUnCordon, karmadaConfig, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
|
@ -139,11 +142,11 @@ func NewCordonHelper(cluster *clusterv1alpha1.Cluster) *CordonHelper {
|
|||
func (c *CordonHelper) UpdateIfRequired(desired int) bool {
|
||||
c.desired = desired
|
||||
|
||||
if desired == desiredCordon && !c.hasUnschedulerTaint() {
|
||||
if desired == DesiredCordon && !c.hasUnschedulerTaint() {
|
||||
return true
|
||||
}
|
||||
|
||||
if desired == desiredUnCordon && c.hasUnschedulerTaint() {
|
||||
if desired == DesiredUnCordon && c.hasUnschedulerTaint() {
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -180,11 +183,11 @@ func (c *CordonHelper) PatchOrReplace(controlPlaneClient *karmadaclientset.Clien
|
|||
Effect: corev1.TaintEffectNoSchedule,
|
||||
}
|
||||
|
||||
if c.desired == desiredCordon {
|
||||
if c.desired == DesiredCordon {
|
||||
c.cluster.Spec.Taints = append(c.cluster.Spec.Taints, unschedulerTaint)
|
||||
}
|
||||
|
||||
if c.desired == desiredUnCordon {
|
||||
if c.desired == DesiredUnCordon {
|
||||
for i, n := 0, len(c.cluster.Spec.Taints); i < n; i++ {
|
||||
if c.cluster.Spec.Taints[i].MatchTaint(&unschedulerTaint) {
|
||||
c.cluster.Spec.Taints[i] = c.cluster.Spec.Taints[n-1]
|
||||
|
|
@ -212,7 +215,7 @@ func (c *CordonHelper) PatchOrReplace(controlPlaneClient *karmadaclientset.Clien
|
|||
// if true cordon cluster otherwise uncordon cluster.
|
||||
func RunCordonOrUncordon(desired int, karmadaConfig KarmadaConfig, opts CommandCordonOption) error {
|
||||
cordonOrUncordon := "cordon"
|
||||
if desired == desiredUnCordon {
|
||||
if desired == DesiredUnCordon {
|
||||
cordonOrUncordon = "un" + cordonOrUncordon
|
||||
}
|
||||
|
||||
|
|
@ -248,7 +251,7 @@ func RunCordonOrUncordon(desired int, karmadaConfig KarmadaConfig, opts CommandC
|
|||
}
|
||||
|
||||
func alreadyStr(desired int) string {
|
||||
if desired == desiredCordon {
|
||||
if desired == DesiredCordon {
|
||||
return "already cordoned"
|
||||
}
|
||||
return "already uncordoned"
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"k8s.io/klog/v2"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
|
||||
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
|
||||
|
|
@ -409,3 +410,161 @@ var _ = framework.SerialDescribe("Karmadactl unjoin testing", ginkgo.Labels{Need
|
|||
})
|
||||
})
|
||||
})
|
||||
|
||||
var _ = framework.SerialDescribe("Karmadactl cordon/uncordon testing", ginkgo.Labels{NeedCreateCluster}, func() {
|
||||
var member1 string
|
||||
var controlPlane string
|
||||
var clusterName string
|
||||
var homeDir string
|
||||
var kubeConfigPath string
|
||||
var clusterContext string
|
||||
var deploymentName, deploymentNamespace string
|
||||
var policyName, policyNamespace string
|
||||
var deployment *appsv1.Deployment
|
||||
var policy *policyv1alpha1.PropagationPolicy
|
||||
var karmadaConfig karmadactl.KarmadaConfig
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
member1 = "member1"
|
||||
clusterName = "member-e2e-" + rand.String(3)
|
||||
homeDir = os.Getenv("HOME")
|
||||
kubeConfigPath = fmt.Sprintf("%s/.kube/%s.config", homeDir, clusterName)
|
||||
controlPlane = fmt.Sprintf("%s-control-plane", clusterName)
|
||||
clusterContext = fmt.Sprintf("kind-%s", clusterName)
|
||||
deploymentName = deploymentNamePrefix + rand.String(RandomStrLength)
|
||||
deploymentNamespace = testNamespace
|
||||
policyName = deploymentName
|
||||
policyNamespace = testNamespace
|
||||
|
||||
deployment = helper.NewDeployment(deploymentNamespace, deploymentName)
|
||||
policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: deployment.APIVersion,
|
||||
Kind: deployment.Kind,
|
||||
Name: deployment.Name,
|
||||
},
|
||||
}, policyv1alpha1.Placement{
|
||||
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
|
||||
ClusterNames: []string{member1, clusterName},
|
||||
},
|
||||
ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{
|
||||
ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDuplicated,
|
||||
},
|
||||
})
|
||||
karmadaConfig = karmadactl.NewKarmadaConfig(clientcmd.NewDefaultPathOptions())
|
||||
})
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
ginkgo.By(fmt.Sprintf("Creating cluster: %s", clusterName), func() {
|
||||
err := createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
})
|
||||
ginkgo.By(fmt.Sprintf("Joinning cluster: %s", clusterName), func() {
|
||||
karmadaConfig := karmadactl.NewKarmadaConfig(clientcmd.NewDefaultPathOptions())
|
||||
opts := karmadactl.CommandJoinOption{
|
||||
GlobalCommandOptions: options.GlobalCommandOptions{
|
||||
DryRun: false,
|
||||
},
|
||||
ClusterNamespace: "karmada-cluster",
|
||||
ClusterName: clusterName,
|
||||
ClusterContext: clusterContext,
|
||||
ClusterKubeConfig: kubeConfigPath,
|
||||
}
|
||||
err := karmadactl.RunJoin(karmadaConfig, opts)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
})
|
||||
// When a newly joined cluster is unready at the beginning, the scheduler will ignore it.
|
||||
ginkgo.By(fmt.Sprintf("wait cluster %s ready", clusterName), func() {
|
||||
framework.WaitClusterFitWith(controlPlaneClient, clusterName, func(cluster *clusterv1alpha1.Cluster) bool {
|
||||
return meta.IsStatusConditionPresentAndEqual(cluster.Status.Conditions, clusterv1alpha1.ClusterConditionReady, metav1.ConditionTrue)
|
||||
})
|
||||
})
|
||||
ginkgo.DeferCleanup(func() {
|
||||
ginkgo.By(fmt.Sprintf("Unjoinning cluster: %s", clusterName), func() {
|
||||
opts := karmadactl.CommandUnjoinOption{
|
||||
GlobalCommandOptions: options.GlobalCommandOptions{
|
||||
DryRun: false,
|
||||
},
|
||||
ClusterNamespace: "karmada-cluster",
|
||||
ClusterName: clusterName,
|
||||
ClusterContext: clusterContext,
|
||||
ClusterKubeConfig: kubeConfigPath,
|
||||
Wait: options.DefaultKarmadactlCommandDuration,
|
||||
}
|
||||
err := karmadactl.RunUnjoin(karmadaConfig, opts)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
})
|
||||
ginkgo.By(fmt.Sprintf("Deleting clusters: %s", clusterName), func() {
|
||||
err := deleteCluster(clusterName, kubeConfigPath)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
_ = os.Remove(kubeConfigPath)
|
||||
})
|
||||
})
|
||||
})
|
||||
ginkgo.Context("cordon cluster", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
opts := karmadactl.CommandCordonOption{
|
||||
GlobalCommandOptions: options.GlobalCommandOptions{
|
||||
DryRun: false,
|
||||
},
|
||||
ClusterName: clusterName,
|
||||
}
|
||||
err := karmadactl.RunCordonOrUncordon(karmadactl.DesiredCordon, karmadaConfig, opts)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
ginkgo.By("cluster should have a taint", func() {
|
||||
clusterObj := &clusterv1alpha1.Cluster{}
|
||||
err := controlPlaneClient.Get(context.TODO(), client.ObjectKey{Name: clusterName}, clusterObj)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
taints := clusterObj.Spec.Taints
|
||||
var unschedulable corev1.Taint
|
||||
for index := range taints {
|
||||
if taints[index].Key == clusterv1alpha1.TaintClusterUnscheduler {
|
||||
unschedulable = taints[index]
|
||||
break
|
||||
}
|
||||
}
|
||||
gomega.Expect(unschedulable).ShouldNot(gomega.BeNil())
|
||||
gomega.Expect(unschedulable.Effect).Should(gomega.Equal(corev1.TaintEffectNoSchedule))
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.It(fmt.Sprintf("deploy deployment should not schedule to cordon cluster %s", clusterName), func() {
|
||||
framework.CreatePropagationPolicy(karmadaClient, policy)
|
||||
framework.CreateDeployment(kubeClient, deployment)
|
||||
ginkgo.DeferCleanup(func() {
|
||||
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
|
||||
framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name)
|
||||
})
|
||||
targetClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
|
||||
gomega.Expect(targetClusters).ShouldNot(gomega.ContainElement(clusterName))
|
||||
})
|
||||
|
||||
ginkgo.It("uncordon cluster", func() {
|
||||
opts := karmadactl.CommandCordonOption{
|
||||
GlobalCommandOptions: options.GlobalCommandOptions{
|
||||
DryRun: false,
|
||||
},
|
||||
ClusterName: clusterName,
|
||||
}
|
||||
err := karmadactl.RunCordonOrUncordon(karmadactl.DesiredUnCordon, karmadaConfig, opts)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
ginkgo.By(fmt.Sprintf("cluster %s taint will be removed", clusterName), func() {
|
||||
clusterObj := &clusterv1alpha1.Cluster{}
|
||||
err := controlPlaneClient.Get(context.TODO(), client.ObjectKey{Name: clusterName}, clusterObj)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
gomega.Expect(clusterObj.Spec.Taints).Should(gomega.BeEmpty())
|
||||
})
|
||||
|
||||
framework.CreatePropagationPolicy(karmadaClient, policy)
|
||||
framework.CreateDeployment(kubeClient, deployment)
|
||||
ginkgo.DeferCleanup(func() {
|
||||
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
|
||||
framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name)
|
||||
})
|
||||
ginkgo.By(fmt.Sprintf("deployment will schedule to cluster %s", clusterName), func() {
|
||||
targetClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
|
||||
gomega.Expect(targetClusters).Should(gomega.ContainElement(clusterName))
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in New Issue