Merge pull request #4539 from zhzhuang-zju/label

cleanup: remove label when pp or cpp is deleted
This commit is contained in:
karmada-bot 2024-01-17 10:14:39 +08:00 committed by GitHub
commit 697170b904
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 333 additions and 14 deletions

View File

@ -1144,14 +1144,14 @@ func (d *ResourceDetector) HandlePropagationPolicyDeletion(policyNS string, poli
// Must remove the labels from the resource template ahead of ResourceBinding, otherwise might lose the chance
// to do that in a retry loop(in particular, the label was successfully removed from ResourceBinding, but
// resource template not), since the ResourceBinding will not be listed again.
if err := d.CleanupLabels(binding.Spec.Resource, policyv1alpha1.PropagationPolicyNamespaceLabel, policyv1alpha1.PropagationPolicyNameLabel); err != nil {
if err := d.CleanupLabels(binding.Spec.Resource, policyv1alpha1.PropagationPolicyNamespaceLabel, policyv1alpha1.PropagationPolicyNameLabel, policyv1alpha1.PropagationPolicyPermanentIDLabel); err != nil {
klog.Errorf("Failed to clean up label from resource(%s-%s/%s) when propagation policy(%s/%s) removing, error: %v",
binding.Spec.Resource.Kind, binding.Spec.Resource.Namespace, binding.Spec.Resource.Name, policyNS, policyName, err)
return err
}
// Clean up the labels from the reference binding so that the karmada scheduler won't reschedule the binding.
if err := d.CleanupResourceBindingLabels(&rbs.Items[index], policyv1alpha1.PropagationPolicyNamespaceLabel, policyv1alpha1.PropagationPolicyNameLabel); err != nil {
if err := d.CleanupResourceBindingLabels(&rbs.Items[index], policyv1alpha1.PropagationPolicyNamespaceLabel, policyv1alpha1.PropagationPolicyNameLabel, policyv1alpha1.PropagationPolicyPermanentIDLabel); err != nil {
klog.Errorf("Failed to clean up label from resource binding(%s/%s) when propagation policy(%s/%s) removing, error: %v",
binding.Namespace, binding.Name, policyNS, policyName, err)
return err
@ -1181,7 +1181,7 @@ func (d *ResourceDetector) HandleClusterPropagationPolicyDeletion(policyName str
// Must remove the labels from the resource template ahead of ClusterResourceBinding, otherwise might lose the chance
// to do that in a retry loop(in particular, the label was successfully removed from ClusterResourceBinding, but
// resource template not), since the ClusterResourceBinding will not be listed again.
if err := d.CleanupLabels(binding.Spec.Resource, policyv1alpha1.ClusterPropagationPolicyLabel); err != nil {
if err := d.CleanupLabels(binding.Spec.Resource, policyv1alpha1.ClusterPropagationPolicyLabel, policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel); err != nil {
klog.Errorf("Failed to clean up label from resource(%s-%s) when cluster propagation policy(%s) removing, error: %v",
binding.Spec.Resource.Kind, binding.Spec.Resource.Name, policyName, err)
errs = append(errs, err)
@ -1190,7 +1190,7 @@ func (d *ResourceDetector) HandleClusterPropagationPolicyDeletion(policyName str
}
// Clean up the labels from the reference binding so that the karmada scheduler won't reschedule the binding.
if err := d.CleanupClusterResourceBindingLabels(&crbs.Items[index], policyv1alpha1.ClusterPropagationPolicyLabel); err != nil {
if err := d.CleanupClusterResourceBindingLabels(&crbs.Items[index], policyv1alpha1.ClusterPropagationPolicyLabel, policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel); err != nil {
klog.Errorf("Failed to clean up label from cluster resource binding(%s) when cluster propagation policy(%s) removing, error: %v",
binding.Name, policyName, err)
errs = append(errs, err)
@ -1208,7 +1208,7 @@ func (d *ResourceDetector) HandleClusterPropagationPolicyDeletion(policyName str
// Must remove the labels from the resource template ahead of ResourceBinding, otherwise might lose the chance
// to do that in a retry loop(in particular, the label was successfully removed from ResourceBinding, but
// resource template not), since the ResourceBinding will not be listed again.
if err := d.CleanupLabels(binding.Spec.Resource, policyv1alpha1.ClusterPropagationPolicyLabel); err != nil {
if err := d.CleanupLabels(binding.Spec.Resource, policyv1alpha1.ClusterPropagationPolicyLabel, policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel); err != nil {
klog.Errorf("Failed to clean up label from resource(%s-%s/%s) when cluster propagation policy(%s) removing, error: %v",
binding.Spec.Resource.Kind, binding.Spec.Resource.Namespace, binding.Spec.Resource.Name, policyName, err)
errs = append(errs, err)
@ -1217,7 +1217,7 @@ func (d *ResourceDetector) HandleClusterPropagationPolicyDeletion(policyName str
}
// Clean up the labels from the reference binding so that the karmada scheduler won't reschedule the binding.
if err := d.CleanupResourceBindingLabels(&rbs.Items[index], policyv1alpha1.ClusterPropagationPolicyLabel); err != nil {
if err := d.CleanupResourceBindingLabels(&rbs.Items[index], policyv1alpha1.ClusterPropagationPolicyLabel, policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel); err != nil {
klog.Errorf("Failed to clean up label from resource binding(%s/%s) when cluster propagation policy(%s) removing, error: %v",
binding.Namespace, binding.Name, policyName, err)
errs = append(errs, err)

View File

@ -33,6 +33,8 @@ import (
"k8s.io/klog/v2"
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
"github.com/karmada-io/karmada/pkg/util/names"
"github.com/karmada-io/karmada/test/e2e/framework"
testhelper "github.com/karmada-io/karmada/test/helper"
)
@ -645,3 +647,140 @@ var _ = ginkgo.Describe("[ExplicitPriority] propagation testing", func() {
})
})
})
// Delete when delete a clusterPropagationPolicy, and no more clusterPropagationPolicy matches the object, something like
// labels should be cleaned.
var _ = ginkgo.Describe("[Delete] clusterPropagation testing", func() {
ginkgo.Context("delete clusterPropagation and remove the labels from the resource template and reference binding", func() {
var policy *policyv1alpha1.ClusterPropagationPolicy
var deployment *appsv1.Deployment
var targetMember string
ginkgo.BeforeEach(func() {
targetMember = framework.ClusterNames()[0]
policyName := deploymentNamePrefix + rand.String(RandomStrLength)
deployment = testhelper.NewDeployment(testNamespace, policyName+"01")
policy = testhelper.NewClusterPropagationPolicy(policyName, []policyv1alpha1.ResourceSelector{
{
APIVersion: deployment.APIVersion,
Kind: deployment.Kind,
Name: deployment.Name,
}}, policyv1alpha1.Placement{
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
ClusterNames: []string{targetMember},
},
})
})
ginkgo.BeforeEach(func() {
framework.CreateClusterPropagationPolicy(karmadaClient, policy)
framework.CreateDeployment(kubeClient, deployment)
ginkgo.DeferCleanup(func() {
// clusterPropagationPolicy will be removed in subsequent test cases
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
})
gomega.Eventually(func() bool {
bindings, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).List(context.TODO(), metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(labels.Set{
policyv1alpha1.ClusterPropagationPolicyLabel: policy.Name,
}).String(),
})
if err != nil {
return false
}
return len(bindings.Items) != 0
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
})
ginkgo.It("delete ClusterPropagationPolicy and check whether labels are deleted correctly", func() {
framework.RemoveClusterPropagationPolicy(karmadaClient, policy.Name)
framework.WaitDeploymentFitWith(kubeClient, deployment.Namespace, deployment.Name, func(dep *appsv1.Deployment) bool {
if dep.Labels == nil {
return true
}
return dep.Labels[policyv1alpha1.ClusterPropagationPolicyLabel] == "" && dep.Labels[policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel] == ""
})
resourceBindingName := names.GenerateBindingName(deployment.Kind, deployment.Name)
framework.WaitResourceBindingFitWith(karmadaClient, deployment.Namespace, resourceBindingName, func(resourceBinding *workv1alpha2.ResourceBinding) bool {
if resourceBinding.Labels == nil {
return true
}
return resourceBinding.Labels[policyv1alpha1.ClusterPropagationPolicyLabel] == "" && resourceBinding.Labels[policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel] == ""
})
})
})
ginkgo.Context("delete clusterPropagation and remove the labels from the resource template and reference clusterBinding", func() {
var crdGroup string
var randStr string
var crdSpecNames apiextensionsv1.CustomResourceDefinitionNames
var crd *apiextensionsv1.CustomResourceDefinition
var crdPolicy *policyv1alpha1.ClusterPropagationPolicy
ginkgo.BeforeEach(func() {
crdGroup = fmt.Sprintf("example-%s.karmada.io", rand.String(RandomStrLength))
randStr = rand.String(RandomStrLength)
crdSpecNames = apiextensionsv1.CustomResourceDefinitionNames{
Kind: fmt.Sprintf("Foo%s", randStr),
ListKind: fmt.Sprintf("Foo%sList", randStr),
Plural: fmt.Sprintf("foo%ss", randStr),
Singular: fmt.Sprintf("foo%s", randStr),
}
crd = testhelper.NewCustomResourceDefinition(crdGroup, crdSpecNames, apiextensionsv1.ClusterScoped)
crdPolicy = testhelper.NewClusterPropagationPolicy(crd.Name, []policyv1alpha1.ResourceSelector{
{
APIVersion: crd.APIVersion,
Kind: crd.Kind,
Name: crd.Name,
},
}, policyv1alpha1.Placement{
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
ClusterNames: framework.ClusterNames(),
},
})
})
ginkgo.BeforeEach(func() {
framework.CreateClusterPropagationPolicy(karmadaClient, crdPolicy)
framework.CreateCRD(dynamicClient, crd)
ginkgo.DeferCleanup(func() {
// clusterPropagationPolicy will be removed in subsequent test cases
framework.RemoveCRD(dynamicClient, crd.Name)
framework.WaitCRDDisappearedOnClusters(framework.ClusterNames(), crd.Name)
})
gomega.Eventually(func() bool {
bindings, err := karmadaClient.WorkV1alpha2().ClusterResourceBindings().List(context.TODO(), metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(labels.Set{
policyv1alpha1.ClusterPropagationPolicyLabel: crdPolicy.Name,
}).String(),
})
if err != nil {
return false
}
return len(bindings.Items) != 0
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
})
ginkgo.It("delete ClusterPropagationPolicy and check whether labels are deleted correctly", func() {
framework.RemoveClusterPropagationPolicy(karmadaClient, crdPolicy.Name)
framework.WaitCRDFitWith(dynamicClient, crd.Name, func(crd *apiextensionsv1.CustomResourceDefinition) bool {
if crd.Labels == nil {
return true
}
return crd.Labels[policyv1alpha1.ClusterPropagationPolicyLabel] == "" && crd.Labels[policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel] == ""
})
resourceBindingName := names.GenerateBindingName(crd.Kind, crd.Name)
framework.WaitClusterResourceBindingFitWith(karmadaClient, resourceBindingName, func(crb *workv1alpha2.ClusterResourceBinding) bool {
if crb.Labels == nil {
return true
}
return crb.Labels[policyv1alpha1.ClusterPropagationPolicyLabel] == "" && crb.Labels[policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel] == ""
})
})
})
})

View File

@ -24,6 +24,12 @@
| Test the high explicit/low priority/implicit priority propagation for deployment | high explicit/low priority/implicit priority ClusterPropagationPolicy propagation testing | [Configure PropagationPolicy/ClusterPropagationPolicy priority](https://karmada.io/docs/next/userguide/scheduling/resource-propagating#configure-propagationpolicyclusterpropagationpolicy-priority) |
| Test the same explicit priority propagation for deployment | same explicit priority ClusterPropagationPolicy propagation testing | [Choose from same-priority PropagationPolicies](https://karmada.io/docs/next/userguide/scheduling/resource-propagating#choose-from-same-priority-propagationpolicies) |
#### Delete clusterPropagation testing
| Test Case | E2E Describe Text | Comments |
|-----------------------------------------------------|-------------------------------------------------------------------------------------------------|------------|
| Test delete clusterpropagationpolicy for deployment | delete ClusterPropagationPolicy and check whether labels are deleted correctly(namespace scope) | |
| Test delete clusterpropagationpolicy for CRD | delete ClusterPropagationPolicy and check whether labels are deleted correctly(cluster scope) | |
#### TODO
1. May need add the test case when the [deployment updates](https://karmada.io/docs/next/userguide/scheduling/resource-propagating#update-deployment).
2. May need add the test case for the **same implicit priority** propagation.

View File

@ -25,11 +25,12 @@
| Test the same explicit priority propagation for deployment | same explicit priority PropagationPolicy propagation testing | [Choose from same-priority PropagationPolicies](https://karmada.io/docs/next/userguide/scheduling/resource-propagating#choose-from-same-priority-propagationpolicies) |
#### Advanced propagation testing
| Test Case | E2E Describe Text | Comments |
|--------------------------------------------------------------------------|-----------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
| Test add resourceSelector of the propagationPolicy for the deployment | add resourceSelectors item | [Update propagationPolicy](https://karmada.io/docs/next/userguide/scheduling/resource-propagating#update-propagationpolicy) |
| Test update resourceSelector of the propagationPolicy for the deployment | update resourceSelectors item | |
| Test update propagateDeps of the propagationPolicy for the deployment | update policy propagateDeps | |
| Test update placement of the propagationPolicy for the deployment | update policy placement | |
| Test modify the old propagationPolicy to unbind and create a new one | modify the old propagationPolicy to unbind and create a new one | |
| Test delete the old propagationPolicy to unbind and create a new one | delete the old propagationPolicy to unbind and create a new one | |
| Test Case | E2E Describe Text | Comments |
|--------------------------------------------------------------------------|-----------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
| Test add resourceSelector of the propagationPolicy for the deployment | add resourceSelectors item | [Update propagationPolicy](https://karmada.io/docs/next/userguide/scheduling/resource-propagating#update-propagationpolicy) |
| Test update resourceSelector of the propagationPolicy for the deployment | update resourceSelectors item | |
| Test update propagateDeps of the propagationPolicy for the deployment | update policy propagateDeps | |
| Test update placement of the propagationPolicy for the deployment | update policy placement | |
| Test delete propagationpolicy for deployment | delete the propagationPolicy and check whether labels are deleted correctly | |
| Test modify the old propagationPolicy to unbind and create a new one | modify the old propagationPolicy to unbind and create a new one | |
| Test delete the old propagationPolicy to unbind and create a new one | delete the old propagationPolicy to unbind and create a new one | |

View File

@ -0,0 +1,38 @@
/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package framework
import (
"context"
"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
)
// WaitClusterResourceBindingFitWith wait clusterResourceBinding fit with util timeout
func WaitClusterResourceBindingFitWith(client karmada.Interface, name string, fit func(clusterResourceBinding *workv1alpha2.ClusterResourceBinding) bool) {
gomega.Eventually(func() bool {
clusterResourceBinding, err := client.WorkV1alpha2().ClusterResourceBindings().Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return false
}
return fit(clusterResourceBinding)
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
}

View File

@ -94,3 +94,19 @@ func WaitCRDDisappearedOnClusters(clusters []string, crdName string) {
}
})
}
// WaitCRDFitWith wait crd fit with util timeout
func WaitCRDFitWith(client dynamic.Interface, crdName string, fit func(crd *apiextensionsv1.CustomResourceDefinition) bool) {
gomega.Eventually(func() bool {
crd := &apiextensionsv1.CustomResourceDefinition{}
unstructured, err := client.Resource(crdGVR).Get(context.TODO(), crdName, metav1.GetOptions{})
if err != nil {
return false
}
err = helper.ConvertToTypedObject(unstructured, crd)
if err != nil {
return false
}
return fit(crd)
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
}

View File

@ -82,6 +82,17 @@ func WaitDeploymentPresentOnClusterFitWith(cluster, namespace, name string, fit
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
}
// WaitDeploymentFitWith wait deployment sync with fit func.
func WaitDeploymentFitWith(client kubernetes.Interface, namespace, name string, fit func(deployment *appsv1.Deployment) bool) {
gomega.Eventually(func() bool {
dep, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return false
}
return fit(dep)
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
}
// WaitDeploymentPresentOnClustersFitWith wait deployment present on cluster sync with fit func.
func WaitDeploymentPresentOnClustersFitWith(clusters []string, namespace, name string, fit func(deployment *appsv1.Deployment) bool) {
ginkgo.By(fmt.Sprintf("Waiting for deployment(%s/%s) synced on member clusters", namespace, name), func() {

View File

@ -0,0 +1,38 @@
/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package framework
import (
"context"
"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
)
// WaitResourceBindingFitWith wait resourceBinding fit with util timeout
func WaitResourceBindingFitWith(client karmada.Interface, namespace, name string, fit func(resourceBinding *workv1alpha2.ResourceBinding) bool) {
gomega.Eventually(func() bool {
resourceBinding, err := client.WorkV1alpha2().ResourceBindings(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return false
}
return fit(resourceBinding)
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
}

View File

@ -41,6 +41,8 @@ import (
"k8s.io/utils/pointer"
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
"github.com/karmada-io/karmada/pkg/util/names"
"github.com/karmada-io/karmada/test/e2e/framework"
testhelper "github.com/karmada-io/karmada/test/helper"
)
@ -1048,6 +1050,74 @@ var _ = ginkgo.Describe("[AdvancedPropagation] propagation testing", func() {
})
})
ginkgo.Context("Delete the propagationPolicy", func() {
var policy *policyv1alpha1.PropagationPolicy
var deployment *appsv1.Deployment
var targetMember string
ginkgo.BeforeEach(func() {
targetMember = framework.ClusterNames()[0]
policyNamespace := testNamespace
policyName := deploymentNamePrefix + rand.String(RandomStrLength)
deployment = testhelper.NewDeployment(testNamespace, policyName+"01")
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
{
APIVersion: deployment.APIVersion,
Kind: deployment.Kind,
Name: deployment.Name,
}}, policyv1alpha1.Placement{
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
ClusterNames: []string{targetMember},
},
})
})
ginkgo.BeforeEach(func() {
framework.CreatePropagationPolicy(karmadaClient, policy)
framework.CreateDeployment(kubeClient, deployment)
ginkgo.DeferCleanup(func() {
// PropagationPolicy will be removed in subsequent test cases
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
})
gomega.Eventually(func() bool {
bindings, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).List(context.TODO(), metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(labels.Set{
policyv1alpha1.PropagationPolicyNamespaceLabel: policy.Namespace,
policyv1alpha1.PropagationPolicyNameLabel: policy.Name,
}).String(),
})
if err != nil {
return false
}
return len(bindings.Items) != 0
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
})
ginkgo.It("delete the propagationPolicy and check whether labels are deleted correctly", func() {
framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name)
framework.WaitDeploymentFitWith(kubeClient, deployment.Namespace, deployment.Name, func(dep *appsv1.Deployment) bool {
if dep.Labels == nil {
return true
}
return dep.Labels[policyv1alpha1.PropagationPolicyPermanentIDLabel] == "" && dep.Labels[policyv1alpha1.PropagationPolicyNameLabel] == "" &&
dep.Labels[policyv1alpha1.PropagationPolicyNamespaceLabel] == ""
})
resourceBindingName := names.GenerateBindingName(deployment.Kind, deployment.Name)
framework.WaitResourceBindingFitWith(karmadaClient, deployment.Namespace, resourceBindingName, func(resourceBinding *workv1alpha2.ResourceBinding) bool {
if resourceBinding.Labels == nil {
return true
}
return resourceBinding.Labels[policyv1alpha1.PropagationPolicyPermanentIDLabel] == "" && resourceBinding.Labels[policyv1alpha1.PropagationPolicyNameLabel] == "" &&
resourceBinding.Labels[policyv1alpha1.PropagationPolicyNamespaceLabel] == ""
})
})
})
ginkgo.Context("Unbind the old PropagationPolicy and create a new one", func() {
var policy01, policy02 *policyv1alpha1.PropagationPolicy
var configmap *corev1.ConfigMap