diff --git a/test/e2e/deploymentreplicassyncer_test.go b/test/e2e/deploymentreplicassyncer_test.go index 85d822984..d8875dd0b 100644 --- a/test/e2e/deploymentreplicassyncer_test.go +++ b/test/e2e/deploymentreplicassyncer_test.go @@ -18,6 +18,7 @@ package e2e import ( "context" + "sort" "time" "github.com/onsi/ginkgo/v2" @@ -30,81 +31,218 @@ import ( "k8s.io/utils/pointer" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/names" "github.com/karmada-io/karmada/test/e2e/framework" "github.com/karmada-io/karmada/test/helper" ) -var _ = ginkgo.Describe("hpa replicas synchronization testing", func() { - ginkgo.Context("Replicas synchronization testing", func() { - var initReplicas = int32(1) - var policyNamespace, policyName string - var namespace, deploymentName, hpaName string - var deployment *appsv1.Deployment - var hpa *autoscalingv2.HorizontalPodAutoscaler - var policy *policyv1alpha1.PropagationPolicy +var _ = ginkgo.Describe("deployment replicas syncer testing", func() { + var namespace string + var deploymentName, hpaName, policyName, bindingName string + var deployment *appsv1.Deployment + var hpa *autoscalingv2.HorizontalPodAutoscaler + var policy *policyv1alpha1.PropagationPolicy + var targetClusters []string + ginkgo.BeforeEach(func() { + namespace = testNamespace + deploymentName = deploymentNamePrefix + rand.String(RandomStrLength) + hpaName = deploymentName + policyName = deploymentName + bindingName = names.GenerateBindingName(util.DeploymentKind, deploymentName) + + // sort member clusters in increasing order + targetClusters = framework.ClusterNames()[0:2] + sort.Strings(targetClusters) + + deployment = helper.NewDeployment(namespace, deploymentName) + hpa = helper.NewHPA(namespace, hpaName, deploymentName) + hpa.Spec.MinReplicas = pointer.Int32(2) + policy = helper.NewPropagationPolicy(namespace, policyName, []policyv1alpha1.ResourceSelector{ + {APIVersion: deployment.APIVersion, Kind: deployment.Kind, Name: deployment.Name}, + {APIVersion: hpa.APIVersion, Kind: hpa.Kind, Name: hpa.Name}, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: targetClusters, + }, + }) + }) + + ginkgo.JustBeforeEach(func() { + framework.CreatePropagationPolicy(karmadaClient, policy) + framework.CreateDeployment(kubeClient, deployment) + framework.CreateHPA(kubeClient, hpa) + + ginkgo.DeferCleanup(func() { + framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name) + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + framework.RemoveHPA(kubeClient, namespace, hpa.Name) + framework.WaitDeploymentDisappearOnClusters(targetClusters, deployment.Namespace, deployment.Name) + }) + }) + + ginkgo.Context("when policy is Duplicated schedule type", func() { ginkgo.BeforeEach(func() { - policyNamespace = testNamespace - namespace = testNamespace - policyName = deploymentNamePrefix + rand.String(RandomStrLength) - deploymentName = policyName - hpaName = policyName + deployment.Spec.Replicas = pointer.Int32(2) + }) - deployment = helper.NewDeployment(namespace, deploymentName) - deployment.Spec.Replicas = pointer.Int32(initReplicas) - hpa = helper.NewHPA(namespace, hpaName, deploymentName) + // Case 1: Deployment(replicas=2) | Policy(Duplicated, two clusters) | HPA(minReplicas=2) + // Expected result: hpa scaling not take effect in updating spec, manually modify spec have no action. + ginkgo.It("general case combined hpa scaling and manually modify in Duplicated type", func() { + ginkgo.By("step1: propagate each 2 replicas to two clusters", func() { + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{2, 2}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 2) + }) + + ginkgo.By("step2: hpa scale each member cluster replicas from 2 to 3", func() { + framework.UpdateHPAWithMinReplicas(kubeClient, namespace, hpa.Name, 3) + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{3, 3}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 2) + }) + + ginkgo.By("step3: manually add deployment template replicas from 2 to 4", func() { + framework.UpdateDeploymentReplicas(kubeClient, deployment, 4) + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{3, 3}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 4) + }) + + ginkgo.By("step4: manually decrease deployment template replicas from 2 to 1", func() { + framework.UpdateDeploymentReplicas(kubeClient, deployment, 1) + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{3, 3}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 1) + }) + }) + }) + + ginkgo.Context("when policy is Divided schedule type, each cluster have more that one replica", func() { + ginkgo.BeforeEach(func() { + policy.Spec.Placement.ReplicaScheduling = helper.NewStaticWeightPolicyStrategy(targetClusters, []int64{1, 1}) + deployment.Spec.Replicas = pointer.Int32(4) + }) + + // Case 2: Deployment(replicas=4) | Policy(Divided, two clusters 1:1) | HPA(minReplicas=2) + // Expected result: hpa scaling can take effect in updating spec, while manually modify not. + ginkgo.It("general case combined hpa scaling and manually modify in Divided type", func() { + ginkgo.By("step1: propagate 4 replicas to two clusters", func() { + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{2, 2}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 4) + }) + + ginkgo.By("step2: hpa scale each member cluster replicas from 2 to 3", func() { + framework.UpdateHPAWithMinReplicas(kubeClient, namespace, hpa.Name, 3) + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{3, 3}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 6) + }) + + ginkgo.By("step3: manually add deployment template replicas from 6 to 10", func() { + framework.UpdateDeploymentReplicas(kubeClient, deployment, 10) + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{3, 3}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 6) + }) + + ginkgo.By("step4: manually decrease deployment template replicas from 6 to 2", func() { + framework.UpdateDeploymentReplicas(kubeClient, deployment, 2) + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{3, 3}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 6) + }) + }) + }) + + ginkgo.Context("when policy is Divided schedule type, one cluster have no replica", func() { + ginkgo.BeforeEach(func() { + policy.Spec.Placement.ReplicaScheduling = helper.NewStaticWeightPolicyStrategy(targetClusters, []int64{1, 1}) + deployment.Spec.Replicas = pointer.Int32(1) + hpa.Spec.MinReplicas = pointer.Int32(1) + }) + + // Case 3: Deployment(replicas=1) | Policy(Divided, two clusters 1:1) | HPA(minReplicas=1) + // Expected result: manually modify can take effect in updating spec. + ginkgo.It("0/1 case, manually modify replicas from 1 to 2", func() { + ginkgo.By("step1: propagate 1 replicas to two clusters", func() { + assertDeploymentTemplateReplicas(namespace, deploymentName, 1) + }) + + ginkgo.By("step2: manually add deployment template replicas from 1 to 2", func() { + framework.UpdateDeploymentReplicas(kubeClient, deployment, 2) + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{1, 1}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 2) + }) + }) + }) + + ginkgo.Context("when policy is Divided schedule type, remove one cluster's replicas", func() { + ginkgo.BeforeEach(func() { + policy.Spec.Placement.ReplicaScheduling = helper.NewStaticWeightPolicyStrategy(targetClusters, []int64{1, 1}) + deployment.Spec.Replicas = pointer.Int32(2) + hpa.Spec.MinReplicas = pointer.Int32(1) + }) + + // Case 4: Deployment(replicas=2) | Policy(Divided, two clusters 1:1) | HPA(minReplicas=1) + // Expected result: manually modify can take effect in updating spec. + ginkgo.It("0/1 case, manually modify replicas from 2 to 1", func() { + ginkgo.By("step1: propagate 2 replicas to two clusters", func() { + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{1, 1}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 2) + }) + + ginkgo.By("step2: manually add deployment template replicas from 2 to 1", func() { + framework.UpdateDeploymentReplicas(kubeClient, deployment, 1) + framework.WaitResourceBindingFitWith(karmadaClient, namespace, bindingName, func(rb *workv1alpha2.ResourceBinding) bool { + return len(rb.Status.AggregatedStatus) == 1 + }) + assertDeploymentTemplateReplicas(namespace, deploymentName, 1) + }) + }) + }) + + ginkgo.Context("when policy is Divided schedule type, propagate 1 replica but hpa minReplicas is 2", func() { + ginkgo.BeforeEach(func() { + policy.Spec.Placement.ReplicaScheduling = helper.NewStaticWeightPolicyStrategy(targetClusters, []int64{1, 1}) + deployment.Spec.Replicas = pointer.Int32(1) hpa.Spec.MinReplicas = pointer.Int32(2) - - policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ - { - APIVersion: deployment.APIVersion, - Kind: deployment.Kind, - Name: deployment.Name, - }, - { - APIVersion: hpa.APIVersion, - Kind: hpa.Kind, - Name: hpa.Name, - }, - }, policyv1alpha1.Placement{ - ClusterAffinity: &policyv1alpha1.ClusterAffinity{ - ClusterNames: framework.ClusterNames(), - }, - }) }) - ginkgo.BeforeEach(func() { - framework.CreatePropagationPolicy(karmadaClient, policy) - framework.CreateDeployment(kubeClient, deployment) - framework.CreateHPA(kubeClient, hpa) - ginkgo.DeferCleanup(func() { - framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name) - framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) - framework.RemoveHPA(kubeClient, namespace, hpa.Name) - framework.WaitDeploymentDisappearOnClusters(framework.ClusterNames(), deployment.Namespace, deployment.Name) - }) - }) - - ginkgo.It("deployment has been scaled up and synchronized to Karmada", func() { - framework.WaitDeploymentPresentOnClustersFitWith(framework.ClusterNames(), deployment.Namespace, deployment.Name, - func(deployment *appsv1.Deployment) bool { - return true - }) - - framework.WaitDeploymentPresentOnClustersFitWith(framework.ClusterNames(), deployment.Namespace, deployment.Name, - func(deployment *appsv1.Deployment) bool { - return *deployment.Spec.Replicas == initReplicas - }) - - expectedReplicas := initReplicas - gomega.Eventually(func() bool { - deploymentExist, err := kubeClient.AppsV1().Deployments(deployment.Namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) - if err != nil { - return false - } - klog.Infof("got: %d, expect: %d", *deploymentExist.Spec.Replicas, expectedReplicas) - return (*deploymentExist.Spec.Replicas == expectedReplicas) && (deploymentExist.Generation == deploymentExist.Status.ObservedGeneration) - }, time.Minute, pollInterval).Should(gomega.Equal(true)) + // Case 5: Deployment(replicas=1) | Policy(Divided, two clusters 1:1) | HPA(minReplicas=2) + // Expected result: it will go through such a process: + // 1. deployment.spec.replicas=1, actual replicas in member1:member2 = 1:0 + // 2. hpa take effect in member1, so actual replicas in member1:member2 = 2:0 + // 3. deployment template updated to 2/2 + // 4. reschedule, assign replicas to member1:member2 = 1:1 + // 5. member1 replicas is retained, so actual replicas in member1:member2 = 2:1 + // 6. hpa take effect in member2, so replicas becomes member1:member2 = 2:2 + // 7. deployment template updated to 4/4 + ginkgo.It("propagate 1 replica but hpa minReplicas is 2", func() { + assertDeploymentWorkloadReplicas(namespace, deploymentName, targetClusters, []int32{2, 2}) + assertDeploymentTemplateReplicas(namespace, deploymentName, 4) }) }) }) + +// assertDeploymentWorkloadReplicas assert replicas in each member cluster eventually equal to @expectedReplicas +func assertDeploymentWorkloadReplicas(namespace, name string, clusters []string, expectedReplicas []int32) { + gomega.Expect(len(clusters)).Should(gomega.Equal(len(expectedReplicas))) + for i, cluster := range clusters { + if expectedReplicas[i] == 0 { + framework.WaitDeploymentDisappearOnCluster(cluster, namespace, name) + return + } + framework.WaitDeploymentPresentOnClustersFitWith([]string{cluster}, namespace, name, func(deployment *appsv1.Deployment) bool { + klog.Infof("in %s cluster, got: %d, expect: %d", cluster, *deployment.Spec.Replicas, expectedReplicas[i]) + return *deployment.Spec.Replicas == expectedReplicas[i] + }) + } +} + +// assertDeploymentTemplateReplicas assert replicas in template spec eventually equal to @expectedSpecReplicas +func assertDeploymentTemplateReplicas(namespace, name string, expectedSpecReplicas int32) { + gomega.Eventually(func() bool { + deploymentExist, err := kubeClient.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false + } + klog.Infof("template spec replicas, got: %d, expect: %d", *deploymentExist.Spec.Replicas, expectedSpecReplicas) + return (*deploymentExist.Spec.Replicas == expectedSpecReplicas) && (deploymentExist.Generation == deploymentExist.Status.ObservedGeneration) + }, time.Minute, pollInterval).Should(gomega.Equal(true)) +} diff --git a/test/e2e/framework/hpa.go b/test/e2e/framework/hpa.go index 2989e31e0..71834d2a8 100644 --- a/test/e2e/framework/hpa.go +++ b/test/e2e/framework/hpa.go @@ -42,3 +42,15 @@ func RemoveHPA(client kubernetes.Interface, namespace, name string) { gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) }) } + +// UpdateHPAWithMinReplicas update HPA with replicas. +func UpdateHPAWithMinReplicas(client kubernetes.Interface, namespace, name string, minReplicas int32) { + ginkgo.By(fmt.Sprintf("Updating HPA(%s/%s)", namespace, name), func() { + newHPA, err := client.AutoscalingV2().HorizontalPodAutoscalers(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + newHPA.Spec.MinReplicas = &minReplicas + _, err = client.AutoscalingV2().HorizontalPodAutoscalers(namespace).Update(context.TODO(), newHPA, metav1.UpdateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} diff --git a/test/helper/policy.go b/test/helper/policy.go index 0679f55d0..5e56901c3 100644 --- a/test/helper/policy.go +++ b/test/helper/policy.go @@ -17,6 +17,7 @@ limitations under the License. package helper import ( + "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -97,6 +98,26 @@ func NewExplicitPriorityClusterPropagationPolicy(policyName string, rsSelectors } } +// NewStaticWeightPolicyStrategy create static weight policy strategy with specific weights +// e.g: @clusters=[member1, member2], @weights=[1, 1], means static weight `member1:member2=1:1` +func NewStaticWeightPolicyStrategy(clusters []string, weights []int64) *policyv1alpha1.ReplicaSchedulingStrategy { + gomega.Expect(len(clusters)).Should(gomega.Equal(len(weights))) + staticWeightList := make([]policyv1alpha1.StaticClusterWeight, 0) + for i, clusterName := range clusters { + staticWeightList = append(staticWeightList, policyv1alpha1.StaticClusterWeight{ + TargetCluster: policyv1alpha1.ClusterAffinity{ClusterNames: []string{clusterName}}, + Weight: weights[i], + }) + } + return &policyv1alpha1.ReplicaSchedulingStrategy{ + ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDivided, + ReplicaDivisionPreference: policyv1alpha1.ReplicaDivisionPreferenceWeighted, + WeightPreference: &policyv1alpha1.ClusterPreferences{ + StaticWeightList: staticWeightList, + }, + } +} + // NewOverridePolicy will build a OverridePolicy object. func NewOverridePolicy(namespace, policyName string, rsSelectors []policyv1alpha1.ResourceSelector, clusterAffinity policyv1alpha1.ClusterAffinity, overriders policyv1alpha1.Overriders) *policyv1alpha1.OverridePolicy { return &policyv1alpha1.OverridePolicy{