From 189a91bbf92bd2e6b3a5d2c97919e5dfdd2cb272 Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Fri, 12 Jan 2024 16:56:47 +0800 Subject: [PATCH] e2e: add a e2e case to test the scenario where pp and cpp are deleted Signed-off-by: zhzhuang-zju --- test/e2e/clusterpropagationpolicy_test.go | 139 +++++++++++++++++++ test/e2e/framework/clusterresourcebinding.go | 38 +++++ test/e2e/framework/customresourcedefine.go | 16 +++ test/e2e/framework/deployment.go | 11 ++ test/e2e/framework/resourcebinding.go | 38 +++++ test/e2e/propagationpolicy_test.go | 70 ++++++++++ 6 files changed, 312 insertions(+) create mode 100644 test/e2e/framework/clusterresourcebinding.go create mode 100644 test/e2e/framework/resourcebinding.go diff --git a/test/e2e/clusterpropagationpolicy_test.go b/test/e2e/clusterpropagationpolicy_test.go index a092614b6..ff595e36e 100644 --- a/test/e2e/clusterpropagationpolicy_test.go +++ b/test/e2e/clusterpropagationpolicy_test.go @@ -33,6 +33,8 @@ import ( "k8s.io/klog/v2" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + "github.com/karmada-io/karmada/pkg/util/names" "github.com/karmada-io/karmada/test/e2e/framework" testhelper "github.com/karmada-io/karmada/test/helper" ) @@ -645,3 +647,140 @@ var _ = ginkgo.Describe("[ExplicitPriority] propagation testing", func() { }) }) }) + +// Delete when delete a clusterPropagationPolicy, and no more clusterPropagationPolicy matches the object, something like +// labels should be cleaned. +var _ = ginkgo.Describe("[Delete] clusterPropagation testing", func() { + ginkgo.Context("delete clusterPropagation and remove the labels from the resource template and reference binding", func() { + var policy *policyv1alpha1.ClusterPropagationPolicy + var deployment *appsv1.Deployment + var targetMember string + + ginkgo.BeforeEach(func() { + targetMember = framework.ClusterNames()[0] + policyName := deploymentNamePrefix + rand.String(RandomStrLength) + + deployment = testhelper.NewDeployment(testNamespace, policyName+"01") + + policy = testhelper.NewClusterPropagationPolicy(policyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deployment.Name, + }}, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{targetMember}, + }, + }) + }) + + ginkgo.BeforeEach(func() { + framework.CreateClusterPropagationPolicy(karmadaClient, policy) + framework.CreateDeployment(kubeClient, deployment) + ginkgo.DeferCleanup(func() { + // clusterPropagationPolicy will be removed in subsequent test cases + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + }) + + gomega.Eventually(func() bool { + bindings, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labels.Set{ + policyv1alpha1.ClusterPropagationPolicyLabel: policy.Name, + }).String(), + }) + if err != nil { + return false + } + return len(bindings.Items) != 0 + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + }) + + ginkgo.It("delete ClusterPropagationPolicy and check whether labels are deleted correctly", func() { + framework.RemoveClusterPropagationPolicy(karmadaClient, policy.Name) + framework.WaitDeploymentFitWith(kubeClient, deployment.Namespace, deployment.Name, func(dep *appsv1.Deployment) bool { + if dep.Labels == nil { + return true + } + return dep.Labels[policyv1alpha1.ClusterPropagationPolicyLabel] == "" && dep.Labels[policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel] == "" + }) + + resourceBindingName := names.GenerateBindingName(deployment.Kind, deployment.Name) + framework.WaitResourceBindingFitWith(karmadaClient, deployment.Namespace, resourceBindingName, func(resourceBinding *workv1alpha2.ResourceBinding) bool { + if resourceBinding.Labels == nil { + return true + } + return resourceBinding.Labels[policyv1alpha1.ClusterPropagationPolicyLabel] == "" && resourceBinding.Labels[policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel] == "" + }) + }) + }) + + ginkgo.Context("delete clusterPropagation and remove the labels from the resource template and reference clusterBinding", func() { + var crdGroup string + var randStr string + var crdSpecNames apiextensionsv1.CustomResourceDefinitionNames + var crd *apiextensionsv1.CustomResourceDefinition + var crdPolicy *policyv1alpha1.ClusterPropagationPolicy + + ginkgo.BeforeEach(func() { + crdGroup = fmt.Sprintf("example-%s.karmada.io", rand.String(RandomStrLength)) + randStr = rand.String(RandomStrLength) + crdSpecNames = apiextensionsv1.CustomResourceDefinitionNames{ + Kind: fmt.Sprintf("Foo%s", randStr), + ListKind: fmt.Sprintf("Foo%sList", randStr), + Plural: fmt.Sprintf("foo%ss", randStr), + Singular: fmt.Sprintf("foo%s", randStr), + } + crd = testhelper.NewCustomResourceDefinition(crdGroup, crdSpecNames, apiextensionsv1.ClusterScoped) + crdPolicy = testhelper.NewClusterPropagationPolicy(crd.Name, []policyv1alpha1.ResourceSelector{ + { + APIVersion: crd.APIVersion, + Kind: crd.Kind, + Name: crd.Name, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: framework.ClusterNames(), + }, + }) + }) + + ginkgo.BeforeEach(func() { + framework.CreateClusterPropagationPolicy(karmadaClient, crdPolicy) + framework.CreateCRD(dynamicClient, crd) + ginkgo.DeferCleanup(func() { + // clusterPropagationPolicy will be removed in subsequent test cases + framework.RemoveCRD(dynamicClient, crd.Name) + framework.WaitCRDDisappearedOnClusters(framework.ClusterNames(), crd.Name) + }) + gomega.Eventually(func() bool { + bindings, err := karmadaClient.WorkV1alpha2().ClusterResourceBindings().List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labels.Set{ + policyv1alpha1.ClusterPropagationPolicyLabel: crdPolicy.Name, + }).String(), + }) + if err != nil { + return false + } + return len(bindings.Items) != 0 + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + }) + + ginkgo.It("delete ClusterPropagationPolicy and check whether labels are deleted correctly", func() { + framework.RemoveClusterPropagationPolicy(karmadaClient, crdPolicy.Name) + framework.WaitCRDFitWith(dynamicClient, crd.Name, func(crd *apiextensionsv1.CustomResourceDefinition) bool { + if crd.Labels == nil { + return true + } + return crd.Labels[policyv1alpha1.ClusterPropagationPolicyLabel] == "" && crd.Labels[policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel] == "" + }) + + resourceBindingName := names.GenerateBindingName(crd.Kind, crd.Name) + framework.WaitClusterResourceBindingFitWith(karmadaClient, resourceBindingName, func(crb *workv1alpha2.ClusterResourceBinding) bool { + if crb.Labels == nil { + return true + } + return crb.Labels[policyv1alpha1.ClusterPropagationPolicyLabel] == "" && crb.Labels[policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel] == "" + }) + }) + }) +}) diff --git a/test/e2e/framework/clusterresourcebinding.go b/test/e2e/framework/clusterresourcebinding.go new file mode 100644 index 000000000..f8aa53bdc --- /dev/null +++ b/test/e2e/framework/clusterresourcebinding.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// WaitClusterResourceBindingFitWith wait clusterResourceBinding fit with util timeout +func WaitClusterResourceBindingFitWith(client karmada.Interface, name string, fit func(clusterResourceBinding *workv1alpha2.ClusterResourceBinding) bool) { + gomega.Eventually(func() bool { + clusterResourceBinding, err := client.WorkV1alpha2().ClusterResourceBindings().Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false + } + return fit(clusterResourceBinding) + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) +} diff --git a/test/e2e/framework/customresourcedefine.go b/test/e2e/framework/customresourcedefine.go index d1bd5db84..34e2d0b30 100644 --- a/test/e2e/framework/customresourcedefine.go +++ b/test/e2e/framework/customresourcedefine.go @@ -94,3 +94,19 @@ func WaitCRDDisappearedOnClusters(clusters []string, crdName string) { } }) } + +// WaitCRDFitWith wait crd fit with util timeout +func WaitCRDFitWith(client dynamic.Interface, crdName string, fit func(crd *apiextensionsv1.CustomResourceDefinition) bool) { + gomega.Eventually(func() bool { + crd := &apiextensionsv1.CustomResourceDefinition{} + unstructured, err := client.Resource(crdGVR).Get(context.TODO(), crdName, metav1.GetOptions{}) + if err != nil { + return false + } + err = helper.ConvertToTypedObject(unstructured, crd) + if err != nil { + return false + } + return fit(crd) + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) +} diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index 23fa8ea91..91fefb1f9 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -82,6 +82,17 @@ func WaitDeploymentPresentOnClusterFitWith(cluster, namespace, name string, fit }, pollTimeout, pollInterval).Should(gomega.Equal(true)) } +// WaitDeploymentFitWith wait deployment sync with fit func. +func WaitDeploymentFitWith(client kubernetes.Interface, namespace, name string, fit func(deployment *appsv1.Deployment) bool) { + gomega.Eventually(func() bool { + dep, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false + } + return fit(dep) + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) +} + // WaitDeploymentPresentOnClustersFitWith wait deployment present on cluster sync with fit func. func WaitDeploymentPresentOnClustersFitWith(clusters []string, namespace, name string, fit func(deployment *appsv1.Deployment) bool) { ginkgo.By(fmt.Sprintf("Waiting for deployment(%s/%s) synced on member clusters", namespace, name), func() { diff --git a/test/e2e/framework/resourcebinding.go b/test/e2e/framework/resourcebinding.go new file mode 100644 index 000000000..4346eab6b --- /dev/null +++ b/test/e2e/framework/resourcebinding.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// WaitResourceBindingFitWith wait resourceBinding fit with util timeout +func WaitResourceBindingFitWith(client karmada.Interface, namespace, name string, fit func(resourceBinding *workv1alpha2.ResourceBinding) bool) { + gomega.Eventually(func() bool { + resourceBinding, err := client.WorkV1alpha2().ResourceBindings(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false + } + return fit(resourceBinding) + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) +} diff --git a/test/e2e/propagationpolicy_test.go b/test/e2e/propagationpolicy_test.go index ba1892ee1..0697d10e0 100644 --- a/test/e2e/propagationpolicy_test.go +++ b/test/e2e/propagationpolicy_test.go @@ -41,6 +41,8 @@ import ( "k8s.io/utils/pointer" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + "github.com/karmada-io/karmada/pkg/util/names" "github.com/karmada-io/karmada/test/e2e/framework" testhelper "github.com/karmada-io/karmada/test/helper" ) @@ -1048,6 +1050,74 @@ var _ = ginkgo.Describe("[AdvancedPropagation] propagation testing", func() { }) }) + ginkgo.Context("Delete the propagationPolicy", func() { + var policy *policyv1alpha1.PropagationPolicy + var deployment *appsv1.Deployment + var targetMember string + + ginkgo.BeforeEach(func() { + targetMember = framework.ClusterNames()[0] + policyNamespace := testNamespace + policyName := deploymentNamePrefix + rand.String(RandomStrLength) + + deployment = testhelper.NewDeployment(testNamespace, policyName+"01") + + policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deployment.Name, + }}, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{targetMember}, + }, + }) + }) + + ginkgo.BeforeEach(func() { + framework.CreatePropagationPolicy(karmadaClient, policy) + framework.CreateDeployment(kubeClient, deployment) + ginkgo.DeferCleanup(func() { + // PropagationPolicy will be removed in subsequent test cases + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + }) + + gomega.Eventually(func() bool { + bindings, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labels.Set{ + policyv1alpha1.PropagationPolicyNamespaceLabel: policy.Namespace, + policyv1alpha1.PropagationPolicyNameLabel: policy.Name, + }).String(), + }) + if err != nil { + return false + } + return len(bindings.Items) != 0 + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + }) + + ginkgo.It("delete the propagationPolicy and check whether labels are deleted correctly", func() { + framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name) + framework.WaitDeploymentFitWith(kubeClient, deployment.Namespace, deployment.Name, func(dep *appsv1.Deployment) bool { + if dep.Labels == nil { + return true + } + return dep.Labels[policyv1alpha1.PropagationPolicyPermanentIDLabel] == "" && dep.Labels[policyv1alpha1.PropagationPolicyNameLabel] == "" && + dep.Labels[policyv1alpha1.PropagationPolicyNamespaceLabel] == "" + + }) + + resourceBindingName := names.GenerateBindingName(deployment.Kind, deployment.Name) + framework.WaitResourceBindingFitWith(karmadaClient, deployment.Namespace, resourceBindingName, func(resourceBinding *workv1alpha2.ResourceBinding) bool { + if resourceBinding.Labels == nil { + return true + } + return resourceBinding.Labels[policyv1alpha1.PropagationPolicyPermanentIDLabel] == "" && resourceBinding.Labels[policyv1alpha1.PropagationPolicyNameLabel] == "" && + resourceBinding.Labels[policyv1alpha1.PropagationPolicyNamespaceLabel] == "" + }) + }) + }) + ginkgo.Context("Unbind the old PropagationPolicy and create a new one", func() { var policy01, policy02 *policyv1alpha1.PropagationPolicy var configmap *corev1.ConfigMap