diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index 91fefb1f9..4b92f7ead 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -174,6 +174,26 @@ func UpdateDeploymentAnnotations(client kubernetes.Interface, deployment *appsv1 }) } +// AppendDeploymentAnnotations append deployment's annotations. +func AppendDeploymentAnnotations(client kubernetes.Interface, deployment *appsv1.Deployment, annotations map[string]string) { + ginkgo.By(fmt.Sprintf("Appending Deployment(%s/%s)'s annotations to %v", deployment.Namespace, deployment.Name, annotations), func() { + gomega.Eventually(func() error { + deploy, err := client.AppsV1().Deployments(deployment.Namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) + if err != nil { + return err + } + if deploy.Annotations == nil { + deploy.Annotations = make(map[string]string, 0) + } + for k, v := range annotations { + deploy.Annotations[k] = v + } + _, err = client.AppsV1().Deployments(deploy.Namespace).Update(context.TODO(), deploy, metav1.UpdateOptions{}) + return err + }, pollTimeout, pollInterval).ShouldNot(gomega.HaveOccurred()) + }) +} + // UpdateDeploymentLabels update deployment's labels. func UpdateDeploymentLabels(client kubernetes.Interface, deployment *appsv1.Deployment, labels map[string]string) { ginkgo.By(fmt.Sprintf("Updating Deployment(%s/%s)'s labels to %v", deployment.Namespace, deployment.Name, labels), func() { diff --git a/test/e2e/framework/propagationpolicy.go b/test/e2e/framework/propagationpolicy.go index 6be8cb2fa..e45f93547 100644 --- a/test/e2e/framework/propagationpolicy.go +++ b/test/e2e/framework/propagationpolicy.go @@ -23,6 +23,7 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -46,6 +47,22 @@ func RemovePropagationPolicy(client karmada.Interface, namespace, name string) { }) } +// RemovePropagationPolicyIfExist delete PropagationPolicy if it exists with karmada client. +func RemovePropagationPolicyIfExist(client karmada.Interface, namespace, name string) { + ginkgo.By(fmt.Sprintf("Removing PropagationPolicy(%s/%s) if it exists", namespace, name), func() { + _, err := client.PolicyV1alpha1().PropagationPolicies(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return + } + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + } + + err = client.PolicyV1alpha1().PropagationPolicies(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + // PatchPropagationPolicy patch PropagationPolicy with karmada client. func PatchPropagationPolicy(client karmada.Interface, namespace, name string, patch []map[string]interface{}, patchType types.PatchType) { ginkgo.By(fmt.Sprintf("Patching PropagationPolicy(%s/%s)", namespace, name), func() { @@ -68,3 +85,14 @@ func UpdatePropagationPolicyWithSpec(client karmada.Interface, namespace, name s gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) }) } + +// WaitPropagationPolicyFitWith wait PropagationPolicy sync with fit func. +func WaitPropagationPolicyFitWith(client karmada.Interface, namespace, name string, fit func(policy *policyv1alpha1.PropagationPolicy) bool) { + gomega.Eventually(func() bool { + policy, err := client.PolicyV1alpha1().PropagationPolicies(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false + } + return fit(policy) + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) +} diff --git a/test/e2e/lazy_activation_policy_test.go b/test/e2e/lazy_activation_policy_test.go index 39b149c4f..401e35374 100644 --- a/test/e2e/lazy_activation_policy_test.go +++ b/test/e2e/lazy_activation_policy_test.go @@ -17,8 +17,11 @@ limitations under the License. package e2e import ( + "time" + "github.com/onsi/ginkgo/v2" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/rand" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" @@ -26,48 +29,237 @@ import ( testhelper "github.com/karmada-io/karmada/test/helper" ) +const waitIntervalForLazyPolicyTest = 3 * time.Second + +// e2e test for https://github.com/karmada-io/karmada/blob/master/docs/proposals/scheduling/activation-preference/lazy-activation-preference.md#test-plan var _ = ginkgo.Describe("Lazy activation policy testing", func() { - ginkgo.Context("Policy created before resource testing", func() { - var policy *policyv1alpha1.PropagationPolicy - var deployment *appsv1.Deployment - var targetMember string + var namespace string + var deploymentName, configMapName, policyName string + var originalCluster, modifiedCluster string + var deployment *appsv1.Deployment + var configMap *corev1.ConfigMap + var policy *policyv1alpha1.PropagationPolicy - ginkgo.BeforeEach(func() { - targetMember = framework.ClusterNames()[0] - policyNamespace := testNamespace - policyName := deploymentNamePrefix + rand.String(RandomStrLength) + ginkgo.BeforeEach(func() { + namespace = testNamespace + deploymentName = deploymentNamePrefix + rand.String(RandomStrLength) + configMapName = deploymentName + policyName = deploymentName + originalCluster = framework.ClusterNames()[0] + modifiedCluster = framework.ClusterNames()[1] - deployment = testhelper.NewDeployment(testNamespace, policyName) - - policy = testhelper.NewLazyPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ - { - APIVersion: deployment.APIVersion, - Kind: deployment.Kind, - Name: deployment.Name, - }}, policyv1alpha1.Placement{ - ClusterAffinity: &policyv1alpha1.ClusterAffinity{ - ClusterNames: []string{targetMember}, - }, - }) + deployment = testhelper.NewDeployment(namespace, deploymentName) + configMap = testhelper.NewConfigMap(namespace, configMapName, map[string]string{"test": "test"}) + policy = testhelper.NewLazyPropagationPolicy(namespace, policyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deploymentName, + }}, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{originalCluster}, + }, }) + }) - ginkgo.BeforeEach(func() { + ginkgo.Context("1. Policy created before resource", func() { + ginkgo.JustBeforeEach(func() { framework.CreatePropagationPolicy(karmadaClient, policy) + waitPropagatePolicyReconciled(namespace, policyName) framework.CreateDeployment(kubeClient, deployment) - ginkgo.DeferCleanup(func() { - framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) - }) - framework.WaitDeploymentPresentOnClusterFitWith(targetMember, deployment.Namespace, deployment.Name, - func(deployment *appsv1.Deployment) bool { return true }) + ginkgo.DeferCleanup(func() { + framework.RemovePropagationPolicyIfExist(karmadaClient, namespace, policyName) + framework.RemoveDeployment(kubeClient, namespace, deploymentName) + framework.WaitDeploymentDisappearOnCluster(originalCluster, namespace, deploymentName) + }) }) - ginkgo.It("policy created before resource testing", func() { - framework.WaitDeploymentPresentOnClusterFitWith(targetMember, deployment.Namespace, deployment.Name, - func(deployment *appsv1.Deployment) bool { return true }) - framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name) - framework.WaitDeploymentPresentOnClusterFitWith(targetMember, deployment.Namespace, deployment.Name, - func(deployment *appsv1.Deployment) bool { return true }) + // Simple Case 1 (Policy created before resource) + // refer: https://github.com/karmada-io/karmada/blob/master/docs/proposals/scheduling/activation-preference/lazy-activation-preference.md#simple-case-1-policy-created-before-resource + ginkgo.It("Simple Case 1 (Policy created before resource)", func() { + ginkgo.By("step 1: deployment propagate success when policy created before it", func() { + waitDeploymentPresentOnCluster(originalCluster, namespace, deploymentName) + }) + + ginkgo.By("step 2: after policy deleted, deployment still keep previous propagation states", func() { + framework.RemovePropagationPolicy(karmadaClient, namespace, policyName) + // wait to distinguish whether the state will not change or have no time to change + time.Sleep(waitIntervalForLazyPolicyTest) + waitDeploymentPresentOnCluster(originalCluster, namespace, deploymentName) + }) + }) + + ginkgo.Context("Propagate dependencies", func() { + ginkgo.BeforeEach(func() { + policy.Spec.PropagateDeps = true + mountConfigMapToDeployment(deployment, configMapName) + }) + + ginkgo.JustBeforeEach(func() { + framework.CreateConfigMap(kubeClient, configMap) + ginkgo.DeferCleanup(func() { + framework.RemoveConfigMap(kubeClient, namespace, configMapName) + }) + }) + + // Combined Case 5 (Propagate dependencies) + // refer: https://github.com/karmada-io/karmada/blob/master/docs/proposals/scheduling/activation-preference/lazy-activation-preference.md#combined-case-5-propagate-dependencies + ginkgo.It("Combined Case 5 (Propagate dependencies)", func() { + ginkgo.By("step 1: deployment and its dependencies could propagate success.", func() { + waitDeploymentPresentOnCluster(originalCluster, namespace, deploymentName) + waitConfigMapPresentOnCluster(originalCluster, namespace, deploymentName) + }) + + ginkgo.By("step 2: change of lazy policy will not take effect", func() { + changePlacementTargetCluster(policy, modifiedCluster) + // wait to distinguish whether the policy will not take effect or have no time to take effect + time.Sleep(waitIntervalForLazyPolicyTest) + waitDeploymentPresentOnCluster(originalCluster, namespace, deploymentName) + waitConfigMapPresentOnCluster(originalCluster, namespace, deploymentName) + }) + + ginkgo.By("step 3: lazy policy take effect when deployment updated, dependencies can also been propagated.", func() { + updateDeploymentManually(deployment) + waitDeploymentPresentOnCluster(modifiedCluster, namespace, deploymentName) + waitConfigMapPresentOnCluster(modifiedCluster, namespace, deploymentName) + }) + }) + }) + }) + + ginkgo.Context("2. Policy created after resource", func() { + ginkgo.JustBeforeEach(func() { + framework.CreateDeployment(kubeClient, deployment) + waitDeploymentReconciled(namespace, deploymentName) + framework.CreatePropagationPolicy(karmadaClient, policy) + + ginkgo.DeferCleanup(func() { + framework.RemovePropagationPolicy(karmadaClient, namespace, policyName) + framework.RemoveDeployment(kubeClient, namespace, deploymentName) + framework.WaitDeploymentDisappearOnCluster(originalCluster, namespace, deploymentName) + }) + }) + + // Simple Case 2 (Policy created after resource) + // refer: https://github.com/karmada-io/karmada/blob/master/docs/proposals/scheduling/activation-preference/lazy-activation-preference.md#simple-case-2-policy-created-after-resource + ginkgo.It("Simple Case 2 (Policy created after resource)", func() { + ginkgo.By("step1: deployment would not propagate when lazy policy created after deployment", func() { + // wait to distinguish whether the deployment will not propagate or have no time to propagate + time.Sleep(waitIntervalForLazyPolicyTest) + framework.WaitDeploymentDisappearOnCluster(originalCluster, namespace, deploymentName) + }) + + ginkgo.By("step2: resource would propagate when itself updated", func() { + updateDeploymentManually(deployment) + waitDeploymentPresentOnCluster(originalCluster, namespace, deploymentName) + }) + }) + + ginkgo.Context("Propagate dependencies", func() { + ginkgo.BeforeEach(func() { + mountConfigMapToDeployment(deployment, configMapName) + framework.CreateConfigMap(kubeClient, configMap) + ginkgo.DeferCleanup(func() { + framework.RemoveConfigMap(kubeClient, namespace, configMapName) + }) + }) + + // Combined Case 6 (Propagate dependencies) + // refer: https://github.com/karmada-io/karmada/blob/master/docs/proposals/scheduling/activation-preference/lazy-activation-preference.md#combined-case-6-propagate-dependencies + ginkgo.It("Combined Case 6 (Propagate dependencies)", func() { + ginkgo.By("step 1: resources would not propagate when lazy policy created after resources", func() { + // wait to distinguish whether the resource will not propagate or have no time to propagate + time.Sleep(waitIntervalForLazyPolicyTest) + framework.WaitDeploymentDisappearOnCluster(originalCluster, namespace, deploymentName) + framework.WaitConfigMapDisappearOnCluster(originalCluster, namespace, configMapName) + }) + + ginkgo.By("step 2: configMap not propagate with deployment since policy PropagateDeps unset", func() { + updateDeploymentManually(deployment) + waitDeploymentPresentOnCluster(originalCluster, namespace, deploymentName) + framework.WaitConfigMapDisappearOnCluster(originalCluster, namespace, configMapName) + }) + + ginkgo.By("step 3: set PropagateDeps of a lazy policy would not take effect immediately", func() { + setPolicyPropagateDeps(policy) + // wait to distinguish whether the policy will not take effect or have no time to take effect + time.Sleep(waitIntervalForLazyPolicyTest) + waitDeploymentPresentOnCluster(originalCluster, namespace, deploymentName) + framework.WaitConfigMapDisappearOnCluster(originalCluster, namespace, configMapName) + }) + + ginkgo.By("step 4: set PropagateDeps of a lazy policy take effect when deployment itself updated", func() { + updateDeploymentManually(deployment) + waitDeploymentPresentOnCluster(originalCluster, namespace, deploymentName) + waitConfigMapPresentOnCluster(originalCluster, namespace, deploymentName) + }) + }) }) }) }) + +// updateDeploymentManually manually update deployment +func updateDeploymentManually(deployment *appsv1.Deployment) { + framework.AppendDeploymentAnnotations(kubeClient, deployment, map[string]string{"reconcileAt": time.Now().Format(time.RFC3339)}) +} + +// waitDeploymentPresentOnCluster wait deployment present on cluster +func waitDeploymentPresentOnCluster(cluster, namespace, name string) { + framework.WaitDeploymentPresentOnClusterFitWith(cluster, namespace, name, func(_ *appsv1.Deployment) bool { + return true + }) +} + +// waitDeploymentReconciled wait reconciliation of deployment finished +func waitDeploymentReconciled(namespace, name string) { + framework.WaitDeploymentFitWith(kubeClient, namespace, name, func(_ *appsv1.Deployment) bool { + // when applying deployment and policy sequentially, we expect deployment to perform the reconcile process before policy, + // but the order is actually uncertain, so we sleep a while to wait reconciliation of deployment finished. + time.Sleep(waitIntervalForLazyPolicyTest) + return true + }) +} + +// waitPropagatePolicyReconciled wait reconciliation of PropagatePolicy finished +func waitPropagatePolicyReconciled(namespace, name string) { + framework.WaitPropagationPolicyFitWith(karmadaClient, namespace, name, func(_ *policyv1alpha1.PropagationPolicy) bool { + // when applying policy and deployment sequentially, we expect policy to perform the reconcile process before deployment, + // but the order is actually uncertain, so we sleep a while to wait reconciliation of policy finished. + time.Sleep(waitIntervalForLazyPolicyTest) + return true + }) +} + +// waitConfigMapPresentOnCluster wait configmap present on cluster +func waitConfigMapPresentOnCluster(cluster, namespace, name string) { + framework.WaitConfigMapPresentOnClusterFitWith(cluster, namespace, name, func(_ *corev1.ConfigMap) bool { + return true + }) +} + +// mountConfigMapToDeployment mount ConfigMap to Deployment +func mountConfigMapToDeployment(deployment *appsv1.Deployment, configMapName string) { + volumes := []corev1.Volume{{ + Name: "vol-configmap", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }}}}} + deployment.Spec.Template.Spec.Volumes = volumes +} + +// changePlacementTargetCluster change policy target cluster to @modifiedCluster +func changePlacementTargetCluster(policy *policyv1alpha1.PropagationPolicy, modifiedCluster string) { + policySpec := policy.Spec + policySpec.Placement.ClusterAffinity = &policyv1alpha1.ClusterAffinity{ClusterNames: []string{modifiedCluster}} + framework.UpdatePropagationPolicyWithSpec(karmadaClient, policy.Namespace, policy.Name, policySpec) +} + +// setPolicyPropagateDeps set PropagateDeps of policy to true +func setPolicyPropagateDeps(policy *policyv1alpha1.PropagationPolicy) { + policySpec := policy.Spec + policySpec.PropagateDeps = true + framework.UpdatePropagationPolicyWithSpec(karmadaClient, policy.Namespace, policy.Name, policySpec) +}