Merge pull request #4903 from chaosi-zju/fedhpa-e2e
add e2e for FederatedHPA and metrics-adapter
This commit is contained in:
karmada-bot
GitHub
commit
499608fe6b
|
|
@ -0,0 +1,219 @@
|
|||
/*
|
||||
Copyright 2024 The Karmada Authors.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
autoscalingv2 "k8s.io/api/autoscaling/v2"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apimachinery/pkg/util/rand"
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1"
|
||||
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
|
||||
"github.com/karmada-io/karmada/test/e2e/framework"
|
||||
testhelper "github.com/karmada-io/karmada/test/helper"
|
||||
)
|
||||
|
||||
var _ = ginkgo.Describe("testing for FederatedHPA and metrics-adapter", func() {
|
||||
var namespace string
|
||||
var deploymentName, serviceName, cppName, federatedHPAName, pressureJobName string
|
||||
var deployment *appsv1.Deployment
|
||||
var service *corev1.Service
|
||||
var cpp, jobCPP *policyv1alpha1.ClusterPropagationPolicy
|
||||
var federatedHPA *autoscalingv1alpha1.FederatedHPA
|
||||
var pressureJob *batchv1.Job
|
||||
var targetClusters []string
|
||||
var clusterNum int32
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
namespace = testNamespace
|
||||
randomStr := rand.String(RandomStrLength)
|
||||
deploymentName = deploymentNamePrefix + randomStr
|
||||
serviceName = serviceNamePrefix + randomStr
|
||||
cppName = cppNamePrefix + randomStr
|
||||
federatedHPAName = federatedHPANamePrefix + randomStr
|
||||
pressureJobName = jobNamePrefix + randomStr
|
||||
targetClusters = framework.ClusterNames()
|
||||
clusterNum = int32(len(targetClusters))
|
||||
|
||||
deployment = testhelper.NewDeployment(namespace, deploymentName)
|
||||
deployment.Spec.Selector.MatchLabels = map[string]string{"app": deploymentName}
|
||||
deployment.Spec.Template.Labels = map[string]string{"app": deploymentName}
|
||||
deployment.Spec.Template.Spec.Containers[0].Resources = corev1.ResourceRequirements{
|
||||
Limits: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("10m"),
|
||||
corev1.ResourceMemory: resource.MustParse("10Mi"),
|
||||
}}
|
||||
service = testhelper.NewService(namespace, serviceName, corev1.ServiceTypeNodePort)
|
||||
service.Spec.Selector = map[string]string{"app": deploymentName}
|
||||
cpp = testhelper.NewClusterPropagationPolicy(cppName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: deployment.APIVersion,
|
||||
Kind: deployment.Kind,
|
||||
Name: deploymentName,
|
||||
},
|
||||
{
|
||||
APIVersion: service.APIVersion,
|
||||
Kind: service.Kind,
|
||||
Name: serviceName,
|
||||
},
|
||||
}, policyv1alpha1.Placement{
|
||||
ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{
|
||||
ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDuplicated,
|
||||
},
|
||||
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
|
||||
ClusterNames: targetClusters,
|
||||
},
|
||||
})
|
||||
federatedHPA = testhelper.NewFederatedHPA(namespace, federatedHPAName, deploymentName)
|
||||
federatedHPA.Spec.MaxReplicas = 6
|
||||
federatedHPA.Spec.Behavior.ScaleUp = &autoscalingv2.HPAScalingRules{StabilizationWindowSeconds: pointer.Int32(3)}
|
||||
federatedHPA.Spec.Behavior.ScaleDown = &autoscalingv2.HPAScalingRules{StabilizationWindowSeconds: pointer.Int32(3)}
|
||||
federatedHPA.Spec.Metrics[0].Resource.Target.AverageUtilization = pointer.Int32(10)
|
||||
|
||||
pressureJob = testhelper.NewJob(namespace, pressureJobName)
|
||||
pressureJob.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyOnFailure
|
||||
pressureJob.Spec.Template.Spec.Containers = []corev1.Container{
|
||||
{
|
||||
Name: pressureJobName,
|
||||
Image: "alpine:3.19.1",
|
||||
Command: []string{"/bin/sh"},
|
||||
Args: []string{"-c", "apk add curl; while true; do for i in `seq 200`; do curl http://" + serviceName + "." + namespace + ":80; done; sleep 1; done"},
|
||||
},
|
||||
}
|
||||
// pressure job always use Duplicated schedule type to prevent certain member cluster from missing it.
|
||||
jobCPP = testhelper.NewClusterPropagationPolicy(pressureJobName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: pressureJob.APIVersion,
|
||||
Kind: pressureJob.Kind,
|
||||
Name: pressureJobName,
|
||||
},
|
||||
}, policyv1alpha1.Placement{
|
||||
ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{
|
||||
ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDuplicated,
|
||||
},
|
||||
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
|
||||
ClusterNames: targetClusters,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.JustBeforeEach(func() {
|
||||
framework.CreateClusterPropagationPolicy(karmadaClient, cpp)
|
||||
framework.CreateDeployment(kubeClient, deployment)
|
||||
framework.CreateService(kubeClient, service)
|
||||
framework.CreateFederatedHPA(karmadaClient, federatedHPA)
|
||||
|
||||
ginkgo.DeferCleanup(func() {
|
||||
framework.RemoveFederatedHPA(karmadaClient, namespace, federatedHPAName)
|
||||
framework.RemoveService(kubeClient, namespace, serviceName)
|
||||
framework.RemoveDeployment(kubeClient, namespace, deploymentName)
|
||||
framework.RemoveClusterPropagationPolicy(karmadaClient, cppName)
|
||||
})
|
||||
})
|
||||
|
||||
// runPressureJob run a job to pressure the deployment to increase its cpu/mem.
|
||||
var runPressureJob = func() {
|
||||
framework.CreateClusterPropagationPolicy(karmadaClient, jobCPP)
|
||||
framework.CreateJob(kubeClient, pressureJob)
|
||||
framework.WaitJobPresentOnClustersFitWith(targetClusters, namespace, pressureJobName, func(_ *batchv1.Job) bool { return true })
|
||||
ginkgo.DeferCleanup(func() {
|
||||
framework.RemoveJob(kubeClient, namespace, pressureJobName)
|
||||
framework.RemoveClusterPropagationPolicy(karmadaClient, pressureJobName)
|
||||
})
|
||||
}
|
||||
|
||||
// 1. duplicated scheduling
|
||||
ginkgo.Context("FederatedHPA scale Deployment in Duplicated schedule type", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
deployment.Spec.Replicas = pointer.Int32(1)
|
||||
})
|
||||
|
||||
ginkgo.It("do scale when metrics of cpu/mem utilization up in Duplicated schedule type", func() {
|
||||
ginkgo.By("step1: check initial replicas result should equal to cluster number", func() {
|
||||
framework.WaitDeploymentStatus(kubeClient, deployment, clusterNum)
|
||||
})
|
||||
|
||||
ginkgo.By("step2: pressure test the deployment of member clusters to increase the cpu/mem", runPressureJob)
|
||||
|
||||
ginkgo.By("step3: check final replicas result should greater than deployment initial replicas", func() {
|
||||
framework.WaitDeploymentFitWith(kubeClient, namespace, deploymentName, func(deploy *appsv1.Deployment) bool {
|
||||
return *deploy.Spec.Replicas > 1
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// 2. static weight scheduling
|
||||
ginkgo.Context("FederatedHPA scale Deployment in Static Weight schedule type", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
// 1:1:1 static weight scheduling type
|
||||
staticSameWeight := newSliceWithDefaultValue(clusterNum, 1)
|
||||
cpp.Spec.Placement.ReplicaScheduling = testhelper.NewStaticWeightPolicyStrategy(targetClusters, staticSameWeight)
|
||||
})
|
||||
|
||||
ginkgo.It("do scale when metrics of cpu/mem utilization up in static weight schedule type", func() {
|
||||
ginkgo.By("step1: check initial replicas result should equal to deployment initial replicas", func() {
|
||||
framework.WaitDeploymentStatus(kubeClient, deployment, 3)
|
||||
})
|
||||
|
||||
ginkgo.By("step2: pressure test the deployment of member clusters to increase the cpu/mem", runPressureJob)
|
||||
|
||||
ginkgo.By("step3: check final replicas result should greater than deployment initial replicas", func() {
|
||||
framework.WaitDeploymentFitWith(kubeClient, namespace, deploymentName, func(deploy *appsv1.Deployment) bool {
|
||||
return *deploy.Spec.Replicas > 3
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// 3. dynamic weight scheduling
|
||||
ginkgo.Context("FederatedHPA scale Deployment in Dynamic Weight schedule type", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
cpp.Spec.Placement.ReplicaScheduling = &policyv1alpha1.ReplicaSchedulingStrategy{
|
||||
ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDivided,
|
||||
ReplicaDivisionPreference: policyv1alpha1.ReplicaDivisionPreferenceWeighted,
|
||||
WeightPreference: &policyv1alpha1.ClusterPreferences{
|
||||
DynamicWeight: policyv1alpha1.DynamicWeightByAvailableReplicas,
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("do scale when metrics of cpu/mem utilization up in dynamic weight schedule type", func() {
|
||||
ginkgo.By("step1: check initial replicas result should equal to deployment initial replicas", func() {
|
||||
framework.WaitDeploymentStatus(kubeClient, deployment, 3)
|
||||
})
|
||||
|
||||
ginkgo.By("step2: pressure test the deployment of member clusters to increase the cpu/mem", runPressureJob)
|
||||
|
||||
ginkgo.By("step3: check final replicas result should greater than deployment initial replicas", func() {
|
||||
framework.WaitDeploymentFitWith(kubeClient, namespace, deploymentName, func(deploy *appsv1.Deployment) bool {
|
||||
return *deploy.Spec.Replicas > 3
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func newSliceWithDefaultValue(size int32, defaultValue int64) []int64 {
|
||||
slice := make([]int64, size)
|
||||
for i := range slice {
|
||||
slice[i] = defaultValue
|
||||
}
|
||||
return slice
|
||||
}
|
||||
|
|
@ -78,6 +78,8 @@ const (
|
|||
cronFedratedHPANamePrefix = "cronfhpa-"
|
||||
resourceRegistryPrefix = "rr-"
|
||||
mcsNamePrefix = "mcs-"
|
||||
ppNamePrefix = "pp-"
|
||||
cppNamePrefix = "cpp-"
|
||||
|
||||
updateDeploymentReplicas = 2
|
||||
updateStatefulSetReplicas = 2
|
||||
|
|
|
|||
Loading…
Reference in New Issue