From d43a119c0af0752e46fe028d0ab99980356128f1 Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Tue, 23 Jul 2024 16:11:09 +0800 Subject: [PATCH] complete the e2e test case scenarios for clusterpp Signed-off-by: zhzhuang-zju --- test/e2e/clusterpropagationpolicy_test.go | 206 +++++++++++++++++- .../clusterpropagationpolicy_test.md | 21 +- .../e2e/framework/clusterpropagationpolicy.go | 3 +- test/e2e/karmadactl_test.go | 6 + 4 files changed, 224 insertions(+), 12 deletions(-) diff --git a/test/e2e/clusterpropagationpolicy_test.go b/test/e2e/clusterpropagationpolicy_test.go index 582da1bce..164e76d80 100644 --- a/test/e2e/clusterpropagationpolicy_test.go +++ b/test/e2e/clusterpropagationpolicy_test.go @@ -174,6 +174,53 @@ var _ = ginkgo.Describe("[BasicClusterPropagation] propagation testing", func() }) }) }) + + ginkgo.Context("Deployment propagation testing", func() { + var policy *policyv1alpha1.ClusterPropagationPolicy + var deployment *appsv1.Deployment + var targetMember string + + ginkgo.BeforeEach(func() { + targetMember = framework.ClusterNames()[0] + policyName := cppNamePrefix + rand.String(RandomStrLength) + deploymentName := deploymentNamePrefix + rand.String(RandomStrLength) + + deployment = testhelper.NewDeployment(testNamespace, deploymentName) + + policy = testhelper.NewClusterPropagationPolicy(policyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deployment.Name, + }}, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{targetMember}, + }, + }) + }) + + ginkgo.BeforeEach(func() { + framework.CreateClusterPropagationPolicy(karmadaClient, policy) + framework.CreateDeployment(kubeClient, deployment) + ginkgo.DeferCleanup(func() { + framework.RemoveClusterPropagationPolicy(karmadaClient, policy.Name) + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + }) + }) + + ginkgo.It("deployment propagation testing", func() { + framework.WaitDeploymentPresentOnClusterFitWith(targetMember, deployment.Namespace, deployment.Name, + func(d *appsv1.Deployment) bool { + return *d.Spec.Replicas == *deployment.Spec.Replicas + }) + + framework.UpdateDeploymentReplicas(kubeClient, deployment, updateDeploymentReplicas) + framework.WaitDeploymentPresentOnClusterFitWith(targetMember, deployment.Namespace, deployment.Name, + func(deployment *appsv1.Deployment) bool { + return *deployment.Spec.Replicas == updateDeploymentReplicas + }) + }) + }) }) var _ = ginkgo.Describe("[AdvancedClusterPropagation] propagation testing", func() { @@ -499,6 +546,102 @@ var _ = ginkgo.Describe("[AdvancedClusterPropagation] propagation testing", func }) }) +// ImplicitPriority more than one PP matches the object, we should choose the most suitable one. +var _ = ginkgo.Describe("[ImplicitPriority] propagation testing", func() { + ginkgo.Context("priorityMatchName/priorityMatchLabel/priorityMatchAll propagation testing", func() { + var priorityMatchName, priorityMatchLabelSelector, priorityMatchAll string + var deploymentNamespace, deploymentName string + var deployment *appsv1.Deployment + var policyMatchName, policyMatchLabelSelector, policyPriorityMatchAll *policyv1alpha1.ClusterPropagationPolicy + var implicitPriorityLabelKey = "priority" + var implicitPriorityLabelValue = "implicit-priority" + + ginkgo.BeforeEach(func() { + priorityMatchName = deploymentNamePrefix + "match-name" + rand.String(RandomStrLength) + priorityMatchLabelSelector = deploymentNamePrefix + "match-labelselector" + rand.String(RandomStrLength) + priorityMatchAll = deploymentNamePrefix + "match-all" + rand.String(RandomStrLength) + deploymentNamespace = testNamespace + deploymentName = deploymentNamePrefix + rand.String(RandomStrLength) + + deployment = testhelper.NewDeployment(deploymentNamespace, deploymentName) + deployment.SetLabels(map[string]string{implicitPriorityLabelKey: implicitPriorityLabelValue}) + policyMatchName = testhelper.NewClusterPropagationPolicy(priorityMatchName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deployment.Name, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: framework.ClusterNames(), + }, + }) + policyMatchLabelSelector = testhelper.NewClusterPropagationPolicy(priorityMatchLabelSelector, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + LabelSelector: metav1.SetAsLabelSelector(labels.Set{implicitPriorityLabelKey: implicitPriorityLabelValue}), + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: framework.ClusterNames(), + }, + }) + policyPriorityMatchAll = testhelper.NewClusterPropagationPolicy(priorityMatchAll, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: framework.ClusterNames(), + }, + }) + }) + + ginkgo.BeforeEach(func() { + framework.CreateClusterPropagationPolicy(karmadaClient, policyMatchName) + framework.CreateClusterPropagationPolicy(karmadaClient, policyMatchLabelSelector) + framework.CreateClusterPropagationPolicy(karmadaClient, policyPriorityMatchAll) + // after all ClusterPropagationPolicy are created, create deployment + time.Sleep(time.Second * 3) + framework.CreateDeployment(kubeClient, deployment) + ginkgo.DeferCleanup(func() { + // Used to ensure that the remaining resources are deleted when the following test cases fail. + framework.RemoveClusterPropagationPolicy(karmadaClient, policyMatchName.Name) + framework.RemoveClusterPropagationPolicy(karmadaClient, policyMatchLabelSelector.Name) + framework.RemoveClusterPropagationPolicy(karmadaClient, policyPriorityMatchAll.Name) + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + framework.WaitDeploymentDisappearOnClusters(framework.ClusterNames(), deployment.Namespace, deployment.Name) + }) + }) + + ginkgo.It("priorityMatchName/priorityMatchLabel/priorityMatchAll testing", func() { + ginkgo.By("check whether the deployment uses the highest priority propagationPolicy(priorityMatchName)", func() { + defer framework.RemoveClusterPropagationPolicy(karmadaClient, policyMatchName.Name) + framework.WaitDeploymentPresentOnClustersFitWith(framework.ClusterNames(), deployment.Namespace, deployment.Name, + func(deployment *appsv1.Deployment) bool { + return deployment.GetAnnotations()[policyv1alpha1.ClusterPropagationPolicyAnnotation] == priorityMatchName + }) + }) + ginkgo.By("check whether the deployment uses the highest priority propagationPolicy(priorityMatchLabel)", func() { + defer framework.RemoveClusterPropagationPolicy(karmadaClient, policyMatchLabelSelector.Name) + framework.WaitDeploymentPresentOnClustersFitWith(framework.ClusterNames(), deployment.Namespace, deployment.Name, + func(deployment *appsv1.Deployment) bool { + return deployment.GetAnnotations()[policyv1alpha1.ClusterPropagationPolicyAnnotation] == priorityMatchLabelSelector + }) + }) + ginkgo.By("check whether the deployment uses the highest priority propagationPolicy(priorityMatchAll)", func() { + defer framework.RemoveClusterPropagationPolicy(karmadaClient, policyPriorityMatchAll.Name) + framework.WaitDeploymentPresentOnClustersFitWith(framework.ClusterNames(), deployment.Namespace, deployment.Name, + func(deployment *appsv1.Deployment) bool { + return deployment.GetAnnotations()[policyv1alpha1.ClusterPropagationPolicyAnnotation] == priorityMatchAll + }) + }) + }) + }) +}) + // ExplicitPriority more than one CPP matches the object, we should select the one with the highest explicit priority, if the // explicit priority is same, select the one with the highest implicit priority. var _ = ginkgo.Describe("[ExplicitPriority] propagation testing", func() { @@ -662,7 +805,7 @@ var _ = ginkgo.Describe("[Delete] clusterPropagation testing", func() { ginkgo.Context("delete clusterPropagation and remove the labels and annotations from the resource template and reference binding", func() { var policy *policyv1alpha1.ClusterPropagationPolicy var deployment *appsv1.Deployment - var targetMember string + var targetMember, updateMember string ginkgo.BeforeEach(func() { targetMember = framework.ClusterNames()[0] @@ -730,6 +873,38 @@ var _ = ginkgo.Describe("[Delete] clusterPropagation testing", func() { return true }) }) + + ginkgo.It("delete the old ClusterPropagationPolicy to unbind and create a new one", func() { + framework.RemoveClusterPropagationPolicy(karmadaClient, policy.Name) + + policyName01 := deploymentNamePrefix + rand.String(RandomStrLength) + updateMember = framework.ClusterNames()[1] + policy01 := testhelper.NewClusterPropagationPolicy(policyName01, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deployment.Name, + }}, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{updateMember}, + }, + }) + framework.CreateClusterPropagationPolicy(karmadaClient, policy01) + defer framework.RemoveClusterPropagationPolicy(karmadaClient, policyName01) + + framework.WaitDeploymentFitWith(kubeClient, deployment.Namespace, deployment.Name, func(dep *appsv1.Deployment) bool { + return dep.Annotations[policyv1alpha1.ClusterPropagationPolicyAnnotation] == policyName01 + }) + resourceBindingName := names.GenerateBindingName(deployment.Kind, deployment.Name) + framework.WaitResourceBindingFitWith(karmadaClient, deployment.Namespace, resourceBindingName, func(resourceBinding *workv1alpha2.ResourceBinding) bool { + return resourceBinding.Annotations[policyv1alpha1.ClusterPropagationPolicyAnnotation] == policyName01 + }) + + framework.WaitDeploymentDisappearOnCluster(targetMember, deployment.Namespace, deployment.Name) + framework.WaitDeploymentPresentOnClusterFitWith(updateMember, deployment.Namespace, deployment.Name, func(*appsv1.Deployment) bool { + return true + }) + }) }) ginkgo.Context("delete clusterPropagation and remove the labels and annotations from the resource template and reference clusterBinding", func() { @@ -737,7 +912,7 @@ var _ = ginkgo.Describe("[Delete] clusterPropagation testing", func() { var randStr string var crdSpecNames apiextensionsv1.CustomResourceDefinitionNames var crd *apiextensionsv1.CustomResourceDefinition - var crdPolicy *policyv1alpha1.ClusterPropagationPolicy + var crdPolicy, crdPolicy01 *policyv1alpha1.ClusterPropagationPolicy ginkgo.BeforeEach(func() { crdGroup = fmt.Sprintf("example-%s.karmada.io", rand.String(RandomStrLength)) @@ -810,5 +985,32 @@ var _ = ginkgo.Describe("[Delete] clusterPropagation testing", func() { return true }) }) + + ginkgo.It("delete the old ClusterPropagationPolicy to unbind and create a new one", func() { + framework.RemoveClusterPropagationPolicy(karmadaClient, crdPolicy.Name) + + crdPolicy01 = testhelper.NewClusterPropagationPolicy(crd.Name+"02", []policyv1alpha1.ResourceSelector{ + { + APIVersion: crd.APIVersion, + Kind: crd.Kind, + Name: crd.Name, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: framework.ClusterNames(), + }, + }) + framework.CreateClusterPropagationPolicy(karmadaClient, crdPolicy01) + defer framework.RemoveClusterPropagationPolicy(karmadaClient, crdPolicy01.Name) + + framework.WaitCRDFitWith(dynamicClient, crd.Name, func(crd *apiextensionsv1.CustomResourceDefinition) bool { + return crd.Annotations[policyv1alpha1.ClusterPropagationPolicyAnnotation] == crdPolicy01.Name + }) + + resourceBindingName := names.GenerateBindingName(crd.Kind, crd.Name) + framework.WaitClusterResourceBindingFitWith(karmadaClient, resourceBindingName, func(crb *workv1alpha2.ClusterResourceBinding) bool { + return crb.Annotations[policyv1alpha1.ClusterPropagationPolicyAnnotation] == crdPolicy01.Name + }) + }) }) }) diff --git a/test/e2e/coverage_docs/clusterpropagationpolicy_test.md b/test/e2e/coverage_docs/clusterpropagationpolicy_test.md index b89e31886..c910c79eb 100644 --- a/test/e2e/coverage_docs/clusterpropagationpolicy_test.md +++ b/test/e2e/coverage_docs/clusterpropagationpolicy_test.md @@ -6,6 +6,7 @@ | Test the propagation policy for CRD | crd propagation testing | [Resource propagating](https://karmada.io/docs/next/userguide/scheduling/resource-propagating) | | Test the propagation policy for clusterRole | clusterRole propagation testing | | | Test the propagation policy for clusterRoleBinding | clusterRoleBinding propagation testing | | +| Test the propagation policy for deployment | deployment propagation testing | | #### Advanced propagation testing | Test Case | E2E Describe Text | Comments | @@ -18,6 +19,11 @@ | Test update placement of the propagation policy for deployment | update policy placement(namespace scope) | | | Test update placement of the propagation policy for clusterRole | update policy placement(cluster scope) | | +#### ImplicitPriority propagation testing +| Test Case | E2E Describe Text | Comments | +|-------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------|----------| +| Test the priorityMatchName/priorityMatchLabel/priorityMatchAll implicit priority propagation for deployment | priorityMatchName/priorityMatchLabel/priorityMatchAll testing | | + #### ExplicitPriority propagation testing | Test Case | E2E Describe Text | Comments | |----------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -25,12 +31,9 @@ | Test the same explicit priority propagation for deployment | same explicit priority ClusterPropagationPolicy propagation testing | [Choose from same-priority PropagationPolicies](https://karmada.io/docs/next/userguide/scheduling/resource-propagating#choose-from-same-priority-propagationpolicies) | #### Delete clusterPropagation testing -| Test Case | E2E Describe Text | Comments | -|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|------------| -| Test delete clusterpropagationpolicy for deployment | delete ClusterPropagationPolicy and check whether labels and annotations are deleted correctly(namespace scope) | | -| Test delete clusterpropagationpolicy for CRD | delete ClusterPropagationPolicy and check whether labels and annotations are deleted correctly(cluster scope) | | - -#### TODO -1. May need add the test case when the [deployment updates](https://karmada.io/docs/next/userguide/scheduling/resource-propagating#update-deployment). -2. May need add the test case for the **same implicit priority** propagation. -3. May need add the test case for **delete** the clusterPropagationPolicy. +| Test Case | E2E Describe Text | Comments | +|--------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|----------| +| Test delete clusterpropagationpolicy for deployment | delete ClusterPropagationPolicy and check whether labels and annotations are deleted correctly(namespace scope) | | +| Test delete clusterpropagationpolicy for deployment and create a new one | delete the old ClusterPropagationPolicy to unbind and create a new one(namespace scope) | | +| Test delete clusterpropagationpolicy for CRD | delete ClusterPropagationPolicy and check whether labels and annotations are deleted correctly(cluster scope) | | +| Test delete clusterpropagationpolicy for CRD and create a new one | delete the old ClusterPropagationPolicy to unbind and create a new one(cluster scope) | | diff --git a/test/e2e/framework/clusterpropagationpolicy.go b/test/e2e/framework/clusterpropagationpolicy.go index 5676c5830..74a9e3ece 100644 --- a/test/e2e/framework/clusterpropagationpolicy.go +++ b/test/e2e/framework/clusterpropagationpolicy.go @@ -23,6 +23,7 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -42,7 +43,7 @@ func CreateClusterPropagationPolicy(client karmada.Interface, policy *policyv1al func RemoveClusterPropagationPolicy(client karmada.Interface, name string) { ginkgo.By(fmt.Sprintf("Removing ClusterPropagationPolicy(%s)", name), func() { err := client.PolicyV1alpha1().ClusterPropagationPolicies().Delete(context.TODO(), name, metav1.DeleteOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(err == nil || apierrors.IsNotFound(err)).Should(gomega.BeTrue()) }) } diff --git a/test/e2e/karmadactl_test.go b/test/e2e/karmadactl_test.go index 70f577549..ba8661f8f 100644 --- a/test/e2e/karmadactl_test.go +++ b/test/e2e/karmadactl_test.go @@ -522,6 +522,12 @@ var _ = ginkgo.Describe("Karmadactl exec testing", func() { }) ginkgo.It("Test exec command", func() { + waitForPodReady := func(namespace, podName string) { + framework.WaitPodPresentOnClusterFitWith(framework.ClusterNames()[0], namespace, podName, func(pod *corev1.Pod) bool { + return pod.Status.Phase == corev1.PodRunning + }) + } + waitForPodReady(pod.Namespace, pod.Name) framework.WaitPodPresentOnClustersFitWith(framework.ClusterNames(), pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { return pod.Status.Phase == corev1.PodRunning