From c038cdd127e008952257f6d4c9abf934db9ab256 Mon Sep 17 00:00:00 2001 From: hejianpeng Date: Tue, 26 Jul 2022 20:26:51 +0800 Subject: [PATCH] support ClusterOverridePolicy in namespaces_sync_controller Signed-off-by: hejianpeng --- .../app/controllermanager.go | 1 + pkg/controllers/binding/common.go | 5 +- .../namespace/namespace_sync_controller.go | 104 ++++++++++++++++-- test/e2e/clusteroverridepolicy_test.go | 99 +++++++++++++++++ test/e2e/framework/clusteroverridepolicy.go | 29 +++++ test/helper/policy.go | 13 +++ 6 files changed, 242 insertions(+), 9 deletions(-) create mode 100644 test/e2e/clusteroverridepolicy_test.go create mode 100644 test/e2e/framework/clusteroverridepolicy.go diff --git a/cmd/controller-manager/app/controllermanager.go b/cmd/controller-manager/app/controllermanager.go index b38a7c607..dec3c4177 100644 --- a/cmd/controller-manager/app/controllermanager.go +++ b/cmd/controller-manager/app/controllermanager.go @@ -367,6 +367,7 @@ func startNamespaceController(ctx controllerscontext.Context) (enabled bool, err Client: ctx.Mgr.GetClient(), EventRecorder: ctx.Mgr.GetEventRecorderFor(namespace.ControllerName), SkippedPropagatingNamespaces: skippedPropagatingNamespaces, + OverrideManager: ctx.OverrideManager, } if err := namespaceSyncController.SetupWithManager(ctx.Mgr); err != nil { return false, err diff --git a/pkg/controllers/binding/common.go b/pkg/controllers/binding/common.go index dd63409a6..982ce5b43 100644 --- a/pkg/controllers/binding/common.go +++ b/pkg/controllers/binding/common.go @@ -125,7 +125,7 @@ func ensureWork( workLabel := mergeLabel(clonedWorkload, workNamespace, binding, scope) annotations := mergeAnnotations(clonedWorkload, binding, scope) - annotations, err = recordAppliedOverrides(cops, ops, annotations) + annotations, err = RecordAppliedOverrides(cops, ops, annotations) if err != nil { klog.Errorf("failed to record appliedOverrides, Error: %v", err) return err @@ -209,7 +209,8 @@ func mergeAnnotations(workload *unstructured.Unstructured, binding metav1.Object return annotations } -func recordAppliedOverrides(cops *overridemanager.AppliedOverrides, ops *overridemanager.AppliedOverrides, +// RecordAppliedOverrides record applied (cluster) overrides to annotations +func RecordAppliedOverrides(cops *overridemanager.AppliedOverrides, ops *overridemanager.AppliedOverrides, annotations map[string]string) (map[string]string, error) { if annotations == nil { annotations = make(map[string]string) diff --git a/pkg/controllers/namespace/namespace_sync_controller.go b/pkg/controllers/namespace/namespace_sync_controller.go index b8bbeb683..ec5a69211 100644 --- a/pkg/controllers/namespace/namespace_sync_controller.go +++ b/pkg/controllers/namespace/namespace_sync_controller.go @@ -7,6 +7,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" @@ -19,10 +20,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" workv1alpha1 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha1" + "github.com/karmada-io/karmada/pkg/controllers/binding" "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/helper" "github.com/karmada-io/karmada/pkg/util/names" + "github.com/karmada-io/karmada/pkg/util/overridemanager" ) const ( @@ -35,6 +39,7 @@ type Controller struct { client.Client // used to operate Work resources. EventRecorder record.EventRecorder SkippedPropagatingNamespaces map[string]struct{} + OverrideManager overridemanager.OverrideManager } // Reconcile performs a full reconciliation for the object referred to by the Request. @@ -96,6 +101,21 @@ func (c *Controller) buildWorks(namespace *corev1.Namespace, clusters []clusterv } for _, cluster := range clusters { + clonedNamespaced := namespaceObj.DeepCopy() + + // namespace only care about ClusterOverridePolicy + cops, _, err := c.OverrideManager.ApplyOverridePolicies(clonedNamespaced, cluster.Name) + if err != nil { + klog.Errorf("Failed to apply overrides for %s/%s/%s, err is: %v", clonedNamespaced.GetKind(), clonedNamespaced.GetNamespace(), clonedNamespaced.GetName(), err) + return err + } + + annotations, err := binding.RecordAppliedOverrides(cops, nil, nil) + if err != nil { + klog.Errorf("failed to record appliedOverrides, Error: %v", err) + return err + } + workNamespace, err := names.GenerateExecutionSpaceName(cluster.Name) if err != nil { klog.Errorf("Failed to generate execution space name for member cluster %s, err is %v", cluster.Name, err) @@ -110,12 +130,13 @@ func (c *Controller) buildWorks(namespace *corev1.Namespace, clusters []clusterv OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(namespace, namespace.GroupVersionKind()), }, + Annotations: annotations, } - util.MergeLabel(namespaceObj, workv1alpha1.WorkNamespaceLabel, workNamespace) - util.MergeLabel(namespaceObj, workv1alpha1.WorkNameLabel, workName) + util.MergeLabel(clonedNamespaced, workv1alpha1.WorkNamespaceLabel, workNamespace) + util.MergeLabel(clonedNamespaced, workv1alpha1.WorkNameLabel, workName) - if err = helper.CreateOrUpdateWork(c.Client, objectMeta, namespaceObj); err != nil { + if err = helper.CreateOrUpdateWork(c.Client, objectMeta, clonedNamespaced); err != nil { return err } } @@ -124,7 +145,7 @@ func (c *Controller) buildWorks(namespace *corev1.Namespace, clusters []clusterv // SetupWithManager creates a controller and register to controller manager. func (c *Controller) SetupWithManager(mgr controllerruntime.Manager) error { - namespaceFn := handler.MapFunc( + clusterNamespaceFn := handler.MapFunc( func(a client.Object) []reconcile.Request { var requests []reconcile.Request namespaceList := &corev1.NamespaceList{} @@ -141,7 +162,7 @@ func (c *Controller) SetupWithManager(mgr controllerruntime.Manager) error { return requests }) - predicate := builder.WithPredicates(predicate.Funcs{ + clusterPredicate := builder.WithPredicates(predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { return true }, @@ -156,7 +177,76 @@ func (c *Controller) SetupWithManager(mgr controllerruntime.Manager) error { }, }) + clusterOverridePolicyNamespaceFn := handler.MapFunc( + func(obj client.Object) []reconcile.Request { + var requests []reconcile.Request + cop, ok := obj.(*policyv1alpha1.ClusterOverridePolicy) + if !ok { + return requests + } + + selectedNamespaces := sets.NewString() + containsAllNamespace := false + for _, rs := range cop.Spec.ResourceSelectors { + if rs.APIVersion != "v1" || rs.Kind != "Namespace" { + continue + } + + if rs.Name == "" { + containsAllNamespace = true + break + } + + selectedNamespaces.Insert(rs.Name) + } + + if containsAllNamespace { + namespaceList := &corev1.NamespaceList{} + if err := c.Client.List(context.TODO(), namespaceList); err != nil { + klog.Errorf("Failed to list namespace, error: %v", err) + return nil + } + + for _, namespace := range namespaceList.Items { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: namespace.Name, + }}) + } + + return requests + } + + for _, ns := range selectedNamespaces.UnsortedList() { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: ns, + }}) + } + + return requests + }) + + clusterOverridePolicyPredicate := builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return true + }, + DeleteFunc: func(event.DeleteEvent) bool { + return true + }, + GenericFunc: func(event.GenericEvent) bool { + return false + }, + }) + return controllerruntime.NewControllerManagedBy(mgr). - For(&corev1.Namespace{}).Watches(&source.Kind{Type: &clusterv1alpha1.Cluster{}}, handler.EnqueueRequestsFromMapFunc(namespaceFn), - predicate).Complete(c) + For(&corev1.Namespace{}). + Watches(&source.Kind{Type: &clusterv1alpha1.Cluster{}}, + handler.EnqueueRequestsFromMapFunc(clusterNamespaceFn), + clusterPredicate). + Watches(&source.Kind{Type: &policyv1alpha1.ClusterOverridePolicy{}}, + handler.EnqueueRequestsFromMapFunc(clusterOverridePolicyNamespaceFn), + clusterOverridePolicyPredicate). + Complete(c) } diff --git a/test/e2e/clusteroverridepolicy_test.go b/test/e2e/clusteroverridepolicy_test.go new file mode 100644 index 000000000..7a08c6ebb --- /dev/null +++ b/test/e2e/clusteroverridepolicy_test.go @@ -0,0 +1,99 @@ +package e2e + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/klog/v2" + + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + "github.com/karmada-io/karmada/test/e2e/framework" + testhelper "github.com/karmada-io/karmada/test/helper" +) + +var _ = ginkgo.Describe("[BasicClusterOverridePolicy] basic cluster override policy testing", func() { + ginkgo.Context("Namespace propagation testing", func() { + var ( + randNamespace string + ns *corev1.Namespace + namespaceCop *policyv1alpha1.ClusterOverridePolicy + + customLabelKey = "hello" + customLabelVal = "world" + plaintextOverriderValue = fmt.Sprintf("{ \"%s\": \"%s\"}", customLabelKey, customLabelVal) + ) + + ginkgo.BeforeEach(func() { + randNamespace = "cop-test-ns-" + rand.String(RandomStrLength) + ns = testhelper.NewNamespace(randNamespace) + namespaceCop = testhelper.NewClusterOverridePolicyByOverrideRules(ns.Name, + []policyv1alpha1.ResourceSelector{ + { + APIVersion: "v1", + Kind: "Namespace", + Name: ns.Name, + }, + }, + []policyv1alpha1.RuleWithCluster{ + { + TargetCluster: &policyv1alpha1.ClusterAffinity{ + ClusterNames: framework.ClusterNames(), + }, + Overriders: policyv1alpha1.Overriders{ + Plaintext: []policyv1alpha1.PlaintextOverrider{ + { + Path: "/metadata/labels", + Operator: policyv1alpha1.OverriderOpAdd, + Value: apiextensionsv1.JSON{Raw: []byte(plaintextOverriderValue)}, + }, + }, + }, + }, + }) + }) + + ginkgo.BeforeEach(func() { + framework.CreateClusterOverridePolicy(karmadaClient, namespaceCop) + framework.CreateNamespace(kubeClient, ns) + ginkgo.DeferCleanup(func() { + framework.RemoveClusterOverridePolicy(karmadaClient, namespaceCop.Name) + framework.RemoveNamespace(kubeClient, ns.Name) + framework.WaitNamespaceDisappearOnClusters(framework.ClusterNames(), ns.Name) + }) + }) + + ginkgo.It("Namespace testing", func() { + ginkgo.By(fmt.Sprintf("Check if namespace(%s) present on member clusters", ns.Name), func() { + for _, clusterName := range framework.ClusterNames() { + clusterClient := framework.GetClusterClient(clusterName) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + + klog.Infof("Waiting for namespace present on cluster(%s)", clusterName) + gomega.Eventually(func(g gomega.Gomega) (bool, error) { + clusterNs, err := clusterClient.CoreV1().Namespaces().Get(context.TODO(), ns.Name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + + return false, err + } + + v, ok := clusterNs.Labels[customLabelKey] + if ok && v == customLabelVal { + return true, nil + } + return false, nil + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + } + }) + }) + }) +}) diff --git a/test/e2e/framework/clusteroverridepolicy.go b/test/e2e/framework/clusteroverridepolicy.go new file mode 100644 index 000000000..d3776ef97 --- /dev/null +++ b/test/e2e/framework/clusteroverridepolicy.go @@ -0,0 +1,29 @@ +package framework + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// CreateClusterOverridePolicy create ClusterOverridePolicy with karmada client. +func CreateClusterOverridePolicy(client karmada.Interface, policy *policyv1alpha1.ClusterOverridePolicy) { + ginkgo.By(fmt.Sprintf("Creating ClusterOverridePolicy(%s)", policy.Name), func() { + _, err := client.PolicyV1alpha1().ClusterOverridePolicies().Create(context.TODO(), policy, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// RemoveClusterOverridePolicy delete ClusterOverridePolicy with karmada client. +func RemoveClusterOverridePolicy(client karmada.Interface, name string) { + ginkgo.By(fmt.Sprintf("Removing ClusterOverridePolicy(%s)", name), func() { + err := client.PolicyV1alpha1().ClusterOverridePolicies().Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} diff --git a/test/helper/policy.go b/test/helper/policy.go index 85bc23892..291162074 100644 --- a/test/helper/policy.go +++ b/test/helper/policy.go @@ -64,6 +64,19 @@ func NewOverridePolicyByOverrideRules(namespace, policyName string, rsSelectors } } +// NewClusterOverridePolicyByOverrideRules will build a ClusterOverridePolicy object by OverrideRules +func NewClusterOverridePolicyByOverrideRules(policyName string, rsSelectors []policyv1alpha1.ResourceSelector, overrideRules []policyv1alpha1.RuleWithCluster) *policyv1alpha1.ClusterOverridePolicy { + return &policyv1alpha1.ClusterOverridePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: policyName, + }, + Spec: policyv1alpha1.OverrideSpec{ + ResourceSelectors: rsSelectors, + OverrideRules: overrideRules, + }, + } +} + // NewFederatedResourceQuota will build a demo FederatedResourceQuota object. func NewFederatedResourceQuota(ns, name string) *policyv1alpha1.FederatedResourceQuota { return &policyv1alpha1.FederatedResourceQuota{