From 4e97a071b986945262da5dc839d5fae2a96a3646 Mon Sep 17 00:00:00 2001 From: Slawomir Chylek Date: Fri, 23 Feb 2018 15:41:52 +0100 Subject: [PATCH] VPA actuation e2e test for pods stuck at pending. --- vertical-pod-autoscaler/e2e/actuation.go | 143 ++++++++++++++++++ vertical-pod-autoscaler/e2e/common.go | 5 + .../hack/deploy-for-e2e.sh | 4 + vertical-pod-autoscaler/hack/run-e2e.sh | 3 +- 4 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 vertical-pod-autoscaler/e2e/actuation.go diff --git a/vertical-pod-autoscaler/e2e/actuation.go b/vertical-pod-autoscaler/e2e/actuation.go new file mode 100644 index 0000000000..8d900cb563 --- /dev/null +++ b/vertical-pod-autoscaler/e2e/actuation.go @@ -0,0 +1,143 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "fmt" + "time" + + apiv1 "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/poc.autoscaling.k8s.io/v1alpha1" + vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" +) + +var _ = actuationSuiteE2eDescribe("Actuation", func() { + f := framework.NewDefaultFramework("vertical-pod-autoscaling") + + ginkgo.It("stops when pods get pending", func() { + + ginkgo.By("Setting up a hamster deployment") + c := f.ClientSet + ns := f.Namespace.Name + + cpuQuantity := parseQuantityOrDie("100m") + memoryQuantity := parseQuantityOrDie("100Mi") + + d := hamsterDeployment(f, cpuQuantity, memoryQuantity) + d, err := c.ExtensionsV1beta1().Deployments(ns).Create(d) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = framework.WaitForDeploymentComplete(c, d) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Setting up a VPA CRD with ridiculous request") + config, err := framework.LoadConfig() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + vpaCRD := newVPA(f, "hamster-vpa", &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "hamster", + }, + }) + + vpaCRD.Status.Recommendation.ContainerRecommendations = []vpa_types.RecommendedContainerResources{ + { + Name: "hamster", + Target: apiv1.ResourceList{ + apiv1.ResourceCPU: parseQuantityOrDie("9999"), // Request 9999 CPUs to make POD pending + }, + }, + } + + vpaClientSet := vpa_clientset.NewForConfigOrDie(config) + vpaClient := vpaClientSet.PocV1alpha1() + _, err = vpaClient.VerticalPodAutoscalers(ns).Create(vpaCRD) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Waiting for pods to be restarted and stuck pending") + + err = assertPodsPendingForDuration(c, d, 1, 2*time.Minute) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + }) +}) + +// assertPodsPendingForDuration checks that at most pendingPodsNum pods are pending for pendingDuration +func assertPodsPendingForDuration(c clientset.Interface, deployment *extensions.Deployment, pendingPodsNum int, pendingDuration time.Duration) error { + + pendingPods := make(map[string]time.Time) + + err := wait.PollImmediate(pollInterval, pollTimeout+pendingDuration, func() (bool, error) { + var err error + currentPodList, err := framework.GetPodsForDeployment(c, deployment) + if err != nil { + return false, err + } + + missingPods := make(map[string]bool) + for podName := range pendingPods { + missingPods[podName] = true + } + + now := time.Now() + for _, pod := range currentPodList.Items { + delete(missingPods, pod.Name) + switch pod.Status.Phase { + case apiv1.PodPending: + _, ok := pendingPods[pod.Name] + if !ok { + pendingPods[pod.Name] = now + } + default: + delete(pendingPods, pod.Name) + } + } + + for missingPod := range missingPods { + delete(pendingPods, missingPod) + } + + if len(pendingPods) < pendingPodsNum { + return false, nil + } + + if len(pendingPods) > pendingPodsNum { + return false, fmt.Errorf("%v pending pods seen - expecting %v", len(pendingPods), pendingPodsNum) + } + + for p, t := range pendingPods { + fmt.Println("task", now, p, t, now.Sub(t), pendingDuration) + if now.Sub(t) < pendingDuration { + return false, nil + } + } + + return true, nil + }) + + if err != nil { + return fmt.Errorf("Assertion failed for pending pods in %v: %v", deployment.Name, err) + } + return nil +} diff --git a/vertical-pod-autoscaler/e2e/common.go b/vertical-pod-autoscaler/e2e/common.go index ce9b0a0197..5347fc2a22 100644 --- a/vertical-pod-autoscaler/e2e/common.go +++ b/vertical-pod-autoscaler/e2e/common.go @@ -35,6 +35,7 @@ const ( updateComponent = "updater" admissionControllerComponent = "admission-controller" fullVpaSuite = "full-vpa" + actuationSuite = "actuation" pollInterval = framework.Poll pollTimeout = 5 * time.Minute ) @@ -59,6 +60,10 @@ func fullVpaE2eDescribe(name string, body func()) bool { return e2eDescribe(fullVpaSuite, name, body) } +func actuationSuiteE2eDescribe(name string, body func()) bool { + return e2eDescribe(actuationSuite, name, body) +} + func hamsterDeployment(f *framework.Framework, cpuQuantity, memoryQuantity resource.Quantity) *extensions.Deployment { d := framework.NewDeployment("hamster-deployment", 3, map[string]string{"app": "hamster"}, "hamster", "gcr.io/google_containers/ubuntu-slim:0.1", extensions.RollingUpdateDeploymentStrategyType) d.ObjectMeta.Namespace = f.Namespace.Name diff --git a/vertical-pod-autoscaler/hack/deploy-for-e2e.sh b/vertical-pod-autoscaler/hack/deploy-for-e2e.sh index 16acd72f95..55e8b94009 100755 --- a/vertical-pod-autoscaler/hack/deploy-for-e2e.sh +++ b/vertical-pod-autoscaler/hack/deploy-for-e2e.sh @@ -26,6 +26,7 @@ function print_help { echo " - recommender" echo " - updater" echo " - admission-controller" + echo " - actuation" echo " - full-vpa" echo "If component is not specified all above will be started." } @@ -49,6 +50,9 @@ case ${SUITE} in full-vpa) COMPONENTS="recommender updater admission-controller" ;; + actuation) + COMPONENTS="updater admission-controller" + ;; *) print_help exit 1 diff --git a/vertical-pod-autoscaler/hack/run-e2e.sh b/vertical-pod-autoscaler/hack/run-e2e.sh index 6778d1346c..01141c6f24 100755 --- a/vertical-pod-autoscaler/hack/run-e2e.sh +++ b/vertical-pod-autoscaler/hack/run-e2e.sh @@ -26,6 +26,7 @@ function print_help { echo " - recommender" echo " - updater" echo " - admission-controller" + echo " - actuation" echo " - full-vpa" } @@ -42,7 +43,7 @@ fi SUITE=$1 case ${SUITE} in - recommender|updater|admission-controller|full-vpa) + recommender|updater|admission-controller|actuation|full-vpa) ${SCRIPT_ROOT}/hack/vpa-down.sh ${SCRIPT_ROOT}/hack/deploy-for-e2e.sh ${SUITE}