From b3ccd3b565e2b227915cf37b776b22ed944ffb79 Mon Sep 17 00:00:00 2001 From: chaosi-zju Date: Mon, 6 May 2024 16:37:46 +0800 Subject: [PATCH] add e2e for FederatedHPA and metrics-adapter. Signed-off-by: chaosi-zju --- test/e2e/federatedhpa_test.go | 219 ++++++++++++++++++++++++++++++++++ test/e2e/suite_test.go | 2 + 2 files changed, 221 insertions(+) create mode 100644 test/e2e/federatedhpa_test.go diff --git a/test/e2e/federatedhpa_test.go b/test/e2e/federatedhpa_test.go new file mode 100644 index 000000000..e0ec559d4 --- /dev/null +++ b/test/e2e/federatedhpa_test.go @@ -0,0 +1,219 @@ +/* +Copyright 2024 The Karmada Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "github.com/onsi/ginkgo/v2" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/utils/pointer" + + autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1" + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + "github.com/karmada-io/karmada/test/e2e/framework" + testhelper "github.com/karmada-io/karmada/test/helper" +) + +var _ = ginkgo.Describe("testing for FederatedHPA and metrics-adapter", func() { + var namespace string + var deploymentName, serviceName, cppName, federatedHPAName, pressureJobName string + var deployment *appsv1.Deployment + var service *corev1.Service + var cpp, jobCPP *policyv1alpha1.ClusterPropagationPolicy + var federatedHPA *autoscalingv1alpha1.FederatedHPA + var pressureJob *batchv1.Job + var targetClusters []string + var clusterNum int32 + + ginkgo.BeforeEach(func() { + namespace = testNamespace + randomStr := rand.String(RandomStrLength) + deploymentName = deploymentNamePrefix + randomStr + serviceName = serviceNamePrefix + randomStr + cppName = cppNamePrefix + randomStr + federatedHPAName = federatedHPANamePrefix + randomStr + pressureJobName = jobNamePrefix + randomStr + targetClusters = framework.ClusterNames() + clusterNum = int32(len(targetClusters)) + + deployment = testhelper.NewDeployment(namespace, deploymentName) + deployment.Spec.Selector.MatchLabels = map[string]string{"app": deploymentName} + deployment.Spec.Template.Labels = map[string]string{"app": deploymentName} + deployment.Spec.Template.Spec.Containers[0].Resources = corev1.ResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }} + service = testhelper.NewService(namespace, serviceName, corev1.ServiceTypeNodePort) + service.Spec.Selector = map[string]string{"app": deploymentName} + cpp = testhelper.NewClusterPropagationPolicy(cppName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deploymentName, + }, + { + APIVersion: service.APIVersion, + Kind: service.Kind, + Name: serviceName, + }, + }, policyv1alpha1.Placement{ + ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{ + ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDuplicated, + }, + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: targetClusters, + }, + }) + federatedHPA = testhelper.NewFederatedHPA(namespace, federatedHPAName, deploymentName) + federatedHPA.Spec.MaxReplicas = 6 + federatedHPA.Spec.Behavior.ScaleUp = &autoscalingv2.HPAScalingRules{StabilizationWindowSeconds: pointer.Int32(3)} + federatedHPA.Spec.Behavior.ScaleDown = &autoscalingv2.HPAScalingRules{StabilizationWindowSeconds: pointer.Int32(3)} + federatedHPA.Spec.Metrics[0].Resource.Target.AverageUtilization = pointer.Int32(10) + + pressureJob = testhelper.NewJob(namespace, pressureJobName) + pressureJob.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyOnFailure + pressureJob.Spec.Template.Spec.Containers = []corev1.Container{ + { + Name: pressureJobName, + Image: "alpine:3.19.1", + Command: []string{"/bin/sh"}, + Args: []string{"-c", "apk add curl; while true; do for i in `seq 200`; do curl http://" + serviceName + "." + namespace + ":80; done; sleep 1; done"}, + }, + } + // pressure job always use Duplicated schedule type to prevent certain member cluster from missing it. + jobCPP = testhelper.NewClusterPropagationPolicy(pressureJobName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: pressureJob.APIVersion, + Kind: pressureJob.Kind, + Name: pressureJobName, + }, + }, policyv1alpha1.Placement{ + ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{ + ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDuplicated, + }, + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: targetClusters, + }, + }) + }) + + ginkgo.JustBeforeEach(func() { + framework.CreateClusterPropagationPolicy(karmadaClient, cpp) + framework.CreateDeployment(kubeClient, deployment) + framework.CreateService(kubeClient, service) + framework.CreateFederatedHPA(karmadaClient, federatedHPA) + + ginkgo.DeferCleanup(func() { + framework.RemoveFederatedHPA(karmadaClient, namespace, federatedHPAName) + framework.RemoveService(kubeClient, namespace, serviceName) + framework.RemoveDeployment(kubeClient, namespace, deploymentName) + framework.RemoveClusterPropagationPolicy(karmadaClient, cppName) + }) + }) + + // runPressureJob run a job to pressure the deployment to increase its cpu/mem. + var runPressureJob = func() { + framework.CreateClusterPropagationPolicy(karmadaClient, jobCPP) + framework.CreateJob(kubeClient, pressureJob) + framework.WaitJobPresentOnClustersFitWith(targetClusters, namespace, pressureJobName, func(_ *batchv1.Job) bool { return true }) + ginkgo.DeferCleanup(func() { + framework.RemoveJob(kubeClient, namespace, pressureJobName) + framework.RemoveClusterPropagationPolicy(karmadaClient, pressureJobName) + }) + } + + // 1. duplicated scheduling + ginkgo.Context("FederatedHPA scale Deployment in Duplicated schedule type", func() { + ginkgo.BeforeEach(func() { + deployment.Spec.Replicas = pointer.Int32(1) + }) + + ginkgo.It("do scale when metrics of cpu/mem utilization up in Duplicated schedule type", func() { + ginkgo.By("step1: check initial replicas result should equal to cluster number", func() { + framework.WaitDeploymentStatus(kubeClient, deployment, clusterNum) + }) + + ginkgo.By("step2: pressure test the deployment of member clusters to increase the cpu/mem", runPressureJob) + + ginkgo.By("step3: check final replicas result should greater than deployment initial replicas", func() { + framework.WaitDeploymentFitWith(kubeClient, namespace, deploymentName, func(deploy *appsv1.Deployment) bool { + return *deploy.Spec.Replicas > 1 + }) + }) + }) + }) + + // 2. static weight scheduling + ginkgo.Context("FederatedHPA scale Deployment in Static Weight schedule type", func() { + ginkgo.BeforeEach(func() { + // 1:1:1 static weight scheduling type + staticSameWeight := newSliceWithDefaultValue(clusterNum, 1) + cpp.Spec.Placement.ReplicaScheduling = testhelper.NewStaticWeightPolicyStrategy(targetClusters, staticSameWeight) + }) + + ginkgo.It("do scale when metrics of cpu/mem utilization up in static weight schedule type", func() { + ginkgo.By("step1: check initial replicas result should equal to deployment initial replicas", func() { + framework.WaitDeploymentStatus(kubeClient, deployment, 3) + }) + + ginkgo.By("step2: pressure test the deployment of member clusters to increase the cpu/mem", runPressureJob) + + ginkgo.By("step3: check final replicas result should greater than deployment initial replicas", func() { + framework.WaitDeploymentFitWith(kubeClient, namespace, deploymentName, func(deploy *appsv1.Deployment) bool { + return *deploy.Spec.Replicas > 3 + }) + }) + }) + }) + + // 3. dynamic weight scheduling + ginkgo.Context("FederatedHPA scale Deployment in Dynamic Weight schedule type", func() { + ginkgo.BeforeEach(func() { + cpp.Spec.Placement.ReplicaScheduling = &policyv1alpha1.ReplicaSchedulingStrategy{ + ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDivided, + ReplicaDivisionPreference: policyv1alpha1.ReplicaDivisionPreferenceWeighted, + WeightPreference: &policyv1alpha1.ClusterPreferences{ + DynamicWeight: policyv1alpha1.DynamicWeightByAvailableReplicas, + }, + } + }) + + ginkgo.It("do scale when metrics of cpu/mem utilization up in dynamic weight schedule type", func() { + ginkgo.By("step1: check initial replicas result should equal to deployment initial replicas", func() { + framework.WaitDeploymentStatus(kubeClient, deployment, 3) + }) + + ginkgo.By("step2: pressure test the deployment of member clusters to increase the cpu/mem", runPressureJob) + + ginkgo.By("step3: check final replicas result should greater than deployment initial replicas", func() { + framework.WaitDeploymentFitWith(kubeClient, namespace, deploymentName, func(deploy *appsv1.Deployment) bool { + return *deploy.Spec.Replicas > 3 + }) + }) + }) + }) +}) + +func newSliceWithDefaultValue(size int32, defaultValue int64) []int64 { + slice := make([]int64, size) + for i := range slice { + slice[i] = defaultValue + } + return slice +} diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index cdd4c4e1c..2338feff3 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -78,6 +78,8 @@ const ( cronFedratedHPANamePrefix = "cronfhpa-" resourceRegistryPrefix = "rr-" mcsNamePrefix = "mcs-" + ppNamePrefix = "pp-" + cppNamePrefix = "cpp-" updateDeploymentReplicas = 2 updateStatefulSetReplicas = 2