Merge pull request #680 from schylek/pending

VPA actuation e2e test for pods stuck at pending.

vertical-pod-autoscaler/e2e/actuation.go

@@ -0,0 +1,143 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package autoscaling
+
+import (
+	"fmt"
+	"time"
+
+	apiv1 "k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/poc.autoscaling.k8s.io/v1alpha1"
+	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+)
+
+var _ = actuationSuiteE2eDescribe("Actuation", func() {
+	f := framework.NewDefaultFramework("vertical-pod-autoscaling")
+
+	ginkgo.It("stops when pods get pending", func() {
+
+		ginkgo.By("Setting up a hamster deployment")
+		c := f.ClientSet
+		ns := f.Namespace.Name
+
+		cpuQuantity := parseQuantityOrDie("100m")
+		memoryQuantity := parseQuantityOrDie("100Mi")
+
+		d := hamsterDeployment(f, cpuQuantity, memoryQuantity)
+		d, err := c.ExtensionsV1beta1().Deployments(ns).Create(d)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		err = framework.WaitForDeploymentComplete(c, d)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Setting up a VPA CRD with ridiculous request")
+		config, err := framework.LoadConfig()
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		vpaCRD := newVPA(f, "hamster-vpa", &metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				"app": "hamster",
+			},
+		})
+
+		vpaCRD.Status.Recommendation.ContainerRecommendations = []vpa_types.RecommendedContainerResources{
+			{
+				Name: "hamster",
+				Target: apiv1.ResourceList{
+					apiv1.ResourceCPU: parseQuantityOrDie("9999"), // Request 9999 CPUs to make POD pending
+				},
+			},
+		}
+
+		vpaClientSet := vpa_clientset.NewForConfigOrDie(config)
+		vpaClient := vpaClientSet.PocV1alpha1()
+		_, err = vpaClient.VerticalPodAutoscalers(ns).Create(vpaCRD)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Waiting for pods to be restarted and stuck pending")
+
+		err = assertPodsPendingForDuration(c, d, 1, 2*time.Minute)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	})
+})
+
+// assertPodsPendingForDuration checks that at most pendingPodsNum pods are pending for pendingDuration
+func assertPodsPendingForDuration(c clientset.Interface, deployment *extensions.Deployment, pendingPodsNum int, pendingDuration time.Duration) error {
+
+	pendingPods := make(map[string]time.Time)
+
+	err := wait.PollImmediate(pollInterval, pollTimeout+pendingDuration, func() (bool, error) {
+		var err error
+		currentPodList, err := framework.GetPodsForDeployment(c, deployment)
+		if err != nil {
+			return false, err
+		}
+
+		missingPods := make(map[string]bool)
+		for podName := range pendingPods {
+			missingPods[podName] = true
+		}
+
+		now := time.Now()
+		for _, pod := range currentPodList.Items {
+			delete(missingPods, pod.Name)
+			switch pod.Status.Phase {
+			case apiv1.PodPending:
+				_, ok := pendingPods[pod.Name]
+				if !ok {
+					pendingPods[pod.Name] = now
+				}
+			default:
+				delete(pendingPods, pod.Name)
+			}
+		}
+
+		for missingPod := range missingPods {
+			delete(pendingPods, missingPod)
+		}
+
+		if len(pendingPods) < pendingPodsNum {
+			return false, nil
+		}
+
+		if len(pendingPods) > pendingPodsNum {
+			return false, fmt.Errorf("%v pending pods seen - expecting %v", len(pendingPods), pendingPodsNum)
+		}
+
+		for p, t := range pendingPods {
+			fmt.Println("task", now, p, t, now.Sub(t), pendingDuration)
+			if now.Sub(t) < pendingDuration {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("Assertion failed for pending pods in %v: %v", deployment.Name, err)
+	}
+	return nil
+}

vertical-pod-autoscaler/e2e/common.go

@@ -35,6 +35,7 @@ const (
 	updateComponent              = "updater"
 	admissionControllerComponent = "admission-controller"
 	fullVpaSuite                 = "full-vpa"
+	actuationSuite               = "actuation"
 	pollInterval                 = framework.Poll
 	pollTimeout                  = 5 * time.Minute
 )
@@ -59,6 +60,10 @@ func fullVpaE2eDescribe(name string, body func()) bool {
 	return e2eDescribe(fullVpaSuite, name, body)
 }
 
+func actuationSuiteE2eDescribe(name string, body func()) bool {
+	return e2eDescribe(actuationSuite, name, body)
+}
+
 func hamsterDeployment(f *framework.Framework, cpuQuantity, memoryQuantity resource.Quantity) *extensions.Deployment {
 	d := framework.NewDeployment("hamster-deployment", 3, map[string]string{"app": "hamster"}, "hamster", "gcr.io/google_containers/ubuntu-slim:0.1", extensions.RollingUpdateDeploymentStrategyType)
 	d.ObjectMeta.Namespace = f.Namespace.Name

vertical-pod-autoscaler/hack/deploy-for-e2e.sh

@@ -26,6 +26,7 @@ function print_help {
   echo " - recommender"
   echo " - updater"
   echo " - admission-controller"
+  echo " - actuation"
   echo " - full-vpa"
   echo "If component is not specified all above will be started."
 }
@@ -49,6 +50,9 @@ case ${SUITE} in
   full-vpa)
     COMPONENTS="recommender updater admission-controller"
     ;;
+  actuation)
+    COMPONENTS="updater admission-controller"
+    ;;
   *)
     print_help
     exit 1

vertical-pod-autoscaler/hack/run-e2e.sh

@@ -26,6 +26,7 @@ function print_help {
   echo " - recommender"
   echo " - updater"
   echo " - admission-controller"
+  echo " - actuation"
   echo " - full-vpa"
 }
 
@@ -42,7 +43,7 @@ fi
 SUITE=$1
 
 case ${SUITE} in
-  recommender|updater|admission-controller|full-vpa)
+  recommender|updater|admission-controller|actuation|full-vpa)
     ${SCRIPT_ROOT}/hack/vpa-down.sh
     ${SCRIPT_ROOT}/hack/deploy-for-e2e.sh ${SUITE}