From e6a68178e924e968138ed7d2349acdfc643339a7 Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Thu, 11 Sep 2025 20:49:58 +0700 Subject: [PATCH] add integration test --- .../e2e/integration/integration.go | 47 ++++++ .../e2e/integration/integration_test.go | 47 ++++++ .../e2e/integration/recommender.go | 138 ++++++++++++++++++ vertical-pod-autoscaler/e2e/utils/common.go | 21 ++- vertical-pod-autoscaler/hack/local-cluster.md | 4 + .../hack/run-integration-locally.sh | 131 +++++++++++++++++ 6 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 vertical-pod-autoscaler/e2e/integration/integration.go create mode 100644 vertical-pod-autoscaler/e2e/integration/integration_test.go create mode 100644 vertical-pod-autoscaler/e2e/integration/recommender.go create mode 100755 vertical-pod-autoscaler/hack/run-integration-locally.sh diff --git a/vertical-pod-autoscaler/e2e/integration/integration.go b/vertical-pod-autoscaler/e2e/integration/integration.go new file mode 100644 index 0000000000..c8e27d187e --- /dev/null +++ b/vertical-pod-autoscaler/e2e/integration/integration.go @@ -0,0 +1,47 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "testing" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + runtimeutils "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/component-base/logs" + "k8s.io/kubernetes/test/e2e/framework" +) + +// RunE2ETests checks configuration parameters (specified through flags) and then runs +// E2E tests using the Ginkgo runner. +// If a "report directory" is specified, one or more JUnit test reports will be +// generated in this directory, and cluster logs will also be saved. +// This function is called on each Ginkgo node in parallel mode. +func RunE2ETests(t *testing.T) { + runtimeutils.ReallyCrash = true + logs.InitLogs() + defer logs.FlushLogs() + + gomega.RegisterFailHandler(framework.Fail) + suiteConfig, _ := ginkgo.GinkgoConfiguration() + // Disable skipped tests unless they are explicitly requested. + if len(suiteConfig.FocusStrings) == 0 && len(suiteConfig.SkipStrings) == 0 { + suiteConfig.SkipStrings = []string{`\[Flaky\]|\[Feature:.+\]`} + } + ginkgo.RunSpecs(t, "Kubernetes e2e suite") +} diff --git a/vertical-pod-autoscaler/e2e/integration/integration_test.go b/vertical-pod-autoscaler/e2e/integration/integration_test.go new file mode 100644 index 0000000000..4cbdd44dba --- /dev/null +++ b/vertical-pod-autoscaler/e2e/integration/integration_test.go @@ -0,0 +1,47 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "flag" + "os" + "testing" + + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" +) + +// handleFlags sets up all flags and parses the command line. +func handleFlags() { + config.CopyFlags(config.Flags, flag.CommandLine) + framework.RegisterCommonFlags(flag.CommandLine) + framework.RegisterClusterFlags(flag.CommandLine) + flag.Parse() +} + +func TestMain(m *testing.M) { + // Register test flags, then parse flags. + handleFlags() + + framework.AfterReadingAllFlags(&framework.TestContext) + + os.Exit(m.Run()) +} + +func TestIntegration(t *testing.T) { + RunE2ETests(t) +} diff --git a/vertical-pod-autoscaler/e2e/integration/recommender.go b/vertical-pod-autoscaler/e2e/integration/recommender.go new file mode 100644 index 0000000000..4ef887e34c --- /dev/null +++ b/vertical-pod-autoscaler/e2e/integration/recommender.go @@ -0,0 +1,138 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils" + + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" + "k8s.io/kubernetes/test/e2e/framework" + podsecurity "k8s.io/pod-security-admission/api" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = utils.RecommenderE2eDescribe("Flags", func() { + f := framework.NewDefaultFramework("vertical-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + var vpaClientSet vpa_clientset.Interface + var hamsterNamespace string + + ginkgo.BeforeEach(func() { + vpaClientSet = utils.GetVpaClientSet(f) + hamsterNamespace = f.Namespace.Name + }) + + ginkgo.AfterEach(func() { + f.ClientSet.AppsV1().Deployments(utils.RecommenderNamespace).Delete(context.TODO(), utils.RecommenderDeploymentName, metav1.DeleteOptions{}) + }) + + ginkgo.It("--vpa-object-namespace", func() { + ginkgo.By("Setting up VPA deployment") + ignoredNamespace, err := f.CreateNamespace(context.TODO(), "ignored-namespace", nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + f.Namespace.Name = utils.RecommenderNamespace + vpaDeployment := utils.NewVPADeployment(f, []string{ + "--recommender-interval=10s", + fmt.Sprintf("--vpa-object-namespace=%s", hamsterNamespace), + }) + utils.StartDeploymentPods(f, vpaDeployment) + + testIncludedAndIgnoredNamespaces(f, vpaClientSet, hamsterNamespace, ignoredNamespace.Name) + }) + + ginkgo.It("--ignored-vpa-object-namespaces", func() { + ginkgo.By("Setting up VPA deployment") + ignoredNamespace, err := f.CreateNamespace(context.TODO(), "ignored-namespace", nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + f.Namespace.Name = utils.RecommenderNamespace + vpaDeployment := utils.NewVPADeployment(f, []string{ + "--recommender-interval=10s", + fmt.Sprintf("--ignored-vpa-object-namespaces=%s", ignoredNamespace.Name), + }) + utils.StartDeploymentPods(f, vpaDeployment) + + testIncludedAndIgnoredNamespaces(f, vpaClientSet, hamsterNamespace, ignoredNamespace.Name) + }) +}) + +// Create VPA and deployment in 2 namespaces, 1 should be ignored +// Ignored namespace VPA and deployment are intentionally created first +// so that by the time included namespace has recommendation generated, +// we know that ignored namespace has been waiting long enough. +func testIncludedAndIgnoredNamespaces(f *framework.Framework, vpaClientSet vpa_clientset.Interface, includedNamespace, ignoredNamespace string) { + ginkgo.By("Setting up a hamster deployment in ignored namespace") + f.Namespace.Name = ignoredNamespace + d := utils.NewNHamstersDeployment(f, 2) + _ = utils.StartDeploymentPods(f, d) + + ginkgo.By("Setting up VPA for ignored namespace") + container1Name := utils.GetHamsterContainerNameByIndex(0) + container2Name := utils.GetHamsterContainerNameByIndex(1) + ignoredVpaCRD := test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(ignoredNamespace). + WithTargetRef(utils.HamsterTargetRef). + WithContainer(container1Name). + WithScalingMode(container1Name, vpa_types.ContainerScalingModeOff). + WithContainer(container2Name). + Get() + f.Namespace.Name = ignoredNamespace + utils.InstallVPA(f, ignoredVpaCRD) + + ginkgo.By("Setting up a hamster deployment in included namespace") + f.Namespace.Name = includedNamespace + d = utils.NewNHamstersDeployment(f, 2) + _ = utils.StartDeploymentPods(f, d) + + ginkgo.By("Setting up VPA for included namespace") + vpaCRD := test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(includedNamespace). + WithTargetRef(utils.HamsterTargetRef). + WithContainer(container1Name). + WithScalingMode(container1Name, vpa_types.ContainerScalingModeOff). + WithContainer(container2Name). + Get() + + f.Namespace.Name = includedNamespace + utils.InstallVPA(f, vpaCRD) + + ginkgo.By("Waiting for recommendation to be filled for just one container") + vpa, err := utils.WaitForRecommendationPresent(vpaClientSet, vpaCRD) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + errMsg := fmt.Sprintf("%s container has recommendations turned off. We expect expect only recommendations for %s", + utils.GetHamsterContainerNameByIndex(0), + utils.GetHamsterContainerNameByIndex(1)) + gomega.Expect(vpa.Status.Recommendation.ContainerRecommendations).Should(gomega.HaveLen(1), errMsg) + gomega.Expect(vpa.Status.Recommendation.ContainerRecommendations[0].ContainerName).To(gomega.Equal(utils.GetHamsterContainerNameByIndex(1)), errMsg) + + ginkgo.By("Ignored namespace should not be recommended") + ignoredVpa, err := vpaClientSet.AutoscalingV1().VerticalPodAutoscalers(ignoredNamespace).Get(context.TODO(), ignoredVpaCRD.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ignoredVpa.Status.Conditions).Should(gomega.HaveLen(0)) +} diff --git a/vertical-pod-autoscaler/e2e/utils/common.go b/vertical-pod-autoscaler/e2e/utils/common.go index a33d7b6cd4..f521fac90b 100644 --- a/vertical-pod-autoscaler/e2e/utils/common.go +++ b/vertical-pod-autoscaler/e2e/utils/common.go @@ -31,10 +31,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" + framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" - framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment" ) const ( @@ -145,6 +145,25 @@ func PatchVpaRecommendation(f *framework.Framework, vpa *vpa_types.VerticalPodAu gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to patch VPA.") } +// NewVPADeployment creates a VPA deployment with n containers +// for e2e test purposes. +func NewVPADeployment(f *framework.Framework, flags []string) *appsv1.Deployment { + d := framework_deployment.NewDeployment( + RecommenderDeploymentName, /*deploymentName*/ + 1, /*replicas*/ + RecommenderLabels, /*podLabels*/ + "recommender", /*imageName*/ + "localhost:5001/vpa-recommender", /*image*/ + appsv1.RollingUpdateDeploymentStrategyType, /*strategyType*/ + ) + d.ObjectMeta.Namespace = f.Namespace.Name + d.Spec.Template.Spec.Containers[0].ImagePullPolicy = apiv1.PullNever // Image must be loaded first + d.Spec.Template.Spec.ServiceAccountName = "vpa-recommender" + d.Spec.Template.Spec.Containers[0].Command = []string{"/recommender"} + d.Spec.Template.Spec.Containers[0].Args = flags + return d +} + // NewNHamstersDeployment creates a simple hamster deployment with n containers // for e2e test purposes. func NewNHamstersDeployment(f *framework.Framework, n int) *appsv1.Deployment { diff --git a/vertical-pod-autoscaler/hack/local-cluster.md b/vertical-pod-autoscaler/hack/local-cluster.md index 5d03c38f61..6264e68c41 100644 --- a/vertical-pod-autoscaler/hack/local-cluster.md +++ b/vertical-pod-autoscaler/hack/local-cluster.md @@ -34,3 +34,7 @@ The local test cases support running the `recommender` with external metrics. T additional permissions we don't want to automatically enable for all customers via the configuration given in `deploy/vpa-rbac.yaml`. The scripts use a context diff `hack/e2e/vpa-rbac.diff` to enable those permission when running locally. + +# Quick Integration Tests + +`run-integration-locally.sh` is a quicker way to integration test compared to `run-e2e-locally.sh`. Only used for simple tests. diff --git a/vertical-pod-autoscaler/hack/run-integration-locally.sh b/vertical-pod-autoscaler/hack/run-integration-locally.sh new file mode 100755 index 0000000000..695d827112 --- /dev/null +++ b/vertical-pod-autoscaler/hack/run-integration-locally.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +# Copyright 2025 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +BASE_NAME=$(basename $0) +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. +source "${SCRIPT_ROOT}/hack/lib/util.sh" + +ARCH=$(kube::util::host_arch) + +function print_help { + echo "ERROR! Usage: $BASE_NAME " + echo " should be one of:" + echo " - recommender" +} + +if [ $# -eq 0 ]; then + print_help + exit 1 +fi + +if [ $# -gt 1 ]; then + print_help + exit 1 +fi + +SUITE=$1 +REQUIRED_COMMANDS=" +docker +go +kind +kubectl +make +" + +for i in $REQUIRED_COMMANDS; do + if ! command -v $i > /dev/null 2>&1 + then + echo "$i could not be found, please ensure it is installed" + echo + echo "The following commands are required to run these tests:" + echo $REQUIRED_COMMANDS + exit 1; + fi +done + +if ! docker ps >/dev/null 2>&1 +then + echo "docker isn't running" + echo + echo "Please ensure that docker is running" + exit 1 +fi + +case ${SUITE} in + recommender) + COMPONENTS="${SUITE}" + ;; + *) + print_help + exit 1 + ;; +esac + +echo "Deleting KIND cluster 'kind'." +kind delete cluster -n kind -q + +echo "Creating KIND cluster 'kind'" +KIND_VERSION="kindest/node:v1.33.0@sha256:02f73d6ae3f11ad5d543f16736a2cb2a63a300ad60e81dac22099b0b04784a4e" +if ! kind create cluster --image=${KIND_VERSION}; then + echo "Failed to create KIND cluster. Exiting. Make sure kind version is updated." + echo "Available versions: https://github.com/kubernetes-sigs/kind/releases" + exit 1 +fi + +# Local KIND images +export REGISTRY=${REGISTRY:-localhost:5001} +export TAG=${TAG:-latest} + +rm -f ${SCRIPT_ROOT}/hack/e2e/vpa-rbac.yaml +patch -c ${SCRIPT_ROOT}/deploy/vpa-rbac.yaml -i ${SCRIPT_ROOT}/hack/e2e/vpa-rbac.diff -o ${SCRIPT_ROOT}/hack/e2e/vpa-rbac.yaml +kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/vpa-rbac.yaml +# Other-versioned CRDs are irrelevant as we're running a modern-ish cluster. +kubectl apply -f ${SCRIPT_ROOT}/deploy/vpa-v1-crd-gen.yaml +kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/k8s-metrics-server.yaml + +for i in ${COMPONENTS}; do + ALL_ARCHITECTURES=${ARCH} make --directory ${SCRIPT_ROOT}/pkg/${i} docker-build REGISTRY=${REGISTRY} TAG=${TAG} + docker tag ${REGISTRY}/vpa-${i}-${ARCH}:${TAG} ${REGISTRY}/vpa-${i}:${TAG} + kind load docker-image ${REGISTRY}/vpa-${i}:${TAG} +done + +export GO111MODULE=on + +case ${SUITE} in + recommender) + + export KUBECONFIG=$HOME/.kube/config + pushd ${SCRIPT_ROOT}/e2e + go test ./integration/*go -v --test.timeout=10m --args --ginkgo.v=true --ginkgo.focus="\[VPA\] \[${SUITE}\]" --disable-log-dump --ginkgo.timeout=10m + INTEGRATION_RESULT=$? + popd + echo integration test result: ${INTEGRATION_RESULT} + if [ $INTEGRATION_RESULT -gt 0 ]; then + echo "Please check integration \"go test\" logs!" + fi + if [ $INTEGRATION_RESULT -gt 0 ]; then + echo "Tests failed" + exit 1 + fi + ;; + *) + print_help + exit 1 + ;; +esac