From af5e4111e3f6fbe2ea4acbb97016641ce0dde96c Mon Sep 17 00:00:00 2001 From: Zhe Jin Date: Thu, 19 Oct 2017 11:07:05 +0800 Subject: [PATCH] preemption integration test and code refine --- .../integration/arbitrator/arbitrator_test.go | 400 ++++++++++++++---- test/integration/framework/master_utils.go | 2 +- 2 files changed, 328 insertions(+), 74 deletions(-) diff --git a/test/integration/arbitrator/arbitrator_test.go b/test/integration/arbitrator/arbitrator_test.go index e5fff0dad..6317159ee 100644 --- a/test/integration/arbitrator/arbitrator_test.go +++ b/test/integration/arbitrator/arbitrator_test.go @@ -39,26 +39,34 @@ import ( restclient "k8s.io/client-go/rest" ) -// prepareNamespace prepare two namespaces "ns01" and "ns02" +// prepareNamespace prepare three namespaces "ns01" "ns02" "ns03" func prepareNamespace(cs *clientset.Clientset) error { - ns01 := &v1.Namespace{ + ns := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: "ns01", - }, - } - ns02 := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ns02", + Name: "xxx", }, } - _, err := cs.CoreV1().Namespaces().Create(ns01) - if err != nil { - return fmt.Errorf("fail to create namespace ns01, %#v", err) + cases := []struct { + name string + }{ + { + name: "ns01", + }, + { + name: "ns02", + }, + { + name: "ns03", + }, } - _, err = cs.CoreV1().Namespaces().Create(ns02) - if err != nil { - return fmt.Errorf("fail to create namespace ns02, %#v", err) + + for _, c := range cases { + ns.Name = c.name + _, err := cs.CoreV1().Namespaces().Create(ns) + if err != nil { + return fmt.Errorf("fail to create namespace %s, %#v", ns.Name, err) + } } return nil @@ -68,16 +76,16 @@ func prepareNamespace(cs *clientset.Clientset) error { func prepareNode(cs *clientset.Clientset) error { node := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ - Name: "node01", - GenerateName: "node01", + Name: "xxx", + GenerateName: "xxx", }, Spec: v1.NodeSpec{ ExternalID: "foo", }, Status: v1.NodeStatus{ Capacity: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("15"), - v1.ResourceMemory: resource.MustParse("15Gi"), + v1.ResourceCPU: resource.MustParse("0"), + v1.ResourceMemory: resource.MustParse("0"), }, Phase: v1.NodeRunning, Conditions: []v1.NodeCondition{ @@ -86,33 +94,43 @@ func prepareNode(cs *clientset.Clientset) error { }, } - _, err := cs.CoreV1().Nodes().Create(node) - if err != nil { - return fmt.Errorf("fail to create node node01, %#v", err) + cases := []struct { + name string + resourceCPU resource.Quantity + resourceMemory resource.Quantity + }{ + { + name: "node01", + resourceCPU: resource.MustParse("15"), + resourceMemory: resource.MustParse("15Gi"), + }, + } + + for _, c := range cases { + node.Name = c.name + node.GenerateName = c.name + node.Status.Capacity[v1.ResourceCPU] = c.resourceCPU + node.Status.Capacity[v1.ResourceMemory] = c.resourceMemory + + _, err := cs.CoreV1().Nodes().Create(node) + if err != nil { + return fmt.Errorf("fail to create node %s, %#v", node.Name, err) + } } return nil } -// prepareResourceQuota prepare two resource quotas "rq01" and "rq02" -// "rq01" is under namespace "ns01", "rq02" is under namespace "ns02" -// "rq01" and "rq02" will not limit cpu and memory usage at first +// prepareResourceQuota prepare three resource quotas "rq01" "rq02" "rq03" +// "rq01" is under namespace "ns01" +// "rq02" is under namespace "ns02" +// "rq03" is under namespace "ns03" +// "rq01" "rq02" "rq03" will not limit cpu and memory usage at first func prepareResourceQuota(cs *clientset.Clientset) error { - rq01 := &v1.ResourceQuota{ + rq := &v1.ResourceQuota{ ObjectMeta: metav1.ObjectMeta{ - Name: "rq01", - Namespace: "ns01", - }, - Spec: v1.ResourceQuotaSpec{ - Hard: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("1000"), - }, - }, - } - rq02 := &v1.ResourceQuota{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rq02", - Namespace: "ns02", + Name: "xxx", + Namespace: "xxx", }, Spec: v1.ResourceQuotaSpec{ Hard: v1.ResourceList{ @@ -121,20 +139,38 @@ func prepareResourceQuota(cs *clientset.Clientset) error { }, } - _, err := cs.CoreV1().ResourceQuotas(rq01.Namespace).Create(rq01) - if err != nil { - return fmt.Errorf("fail to create quota rq01, %#v", err) + cases := []struct { + name string + ns string + }{ + { + name: "rq01", + ns: "ns01", + }, + { + name: "rq02", + ns: "ns02", + }, + { + name: "rq03", + ns: "ns03", + }, } - _, err = cs.CoreV1().ResourceQuotas(rq02.Namespace).Create(rq02) - if err != nil { - return fmt.Errorf("fail to create quota rq02, %#v", err) + + for _, c := range cases { + rq.Name = c.name + rq.Namespace = c.ns + _, err := cs.CoreV1().ResourceQuotas(rq.Namespace).Create(rq) + if err != nil { + return fmt.Errorf("fail to create quota %s, %#v", rq.Name, err) + } } return nil } -// prepareCRD prepare customer resource definition "ResourceQuotaAllocator" -// create two ResourceQuotaAllocator, "queue01" and "queue02" +// prepareCRD prepare customer resource definition "Queue" +// create two Queues, "queue01" and "queue02" // "queue01" is under namespace "ns01" and has attribute "weight=1" // "queue02" is under namespace "ns02" and has attribute "weight=2" func prepareCRD(config *restclient.Config) error { @@ -153,41 +189,211 @@ func prepareCRD(config *restclient.Config) error { return fmt.Errorf("fail to create crd client, %#v", err) } - crd01 := &apiv1.Queue{ + crd := &apiv1.Queue{ ObjectMeta: metav1.ObjectMeta{ - Name: "queue01", - Namespace: "ns01", + Name: "xxx", + Namespace: "xxx", }, Spec: apiv1.QueueSpec{ - Weight: 1, - }, - } - crd02 := &apiv1.Queue{ - ObjectMeta: metav1.ObjectMeta{ - Name: "queue02", - Namespace: "ns02", - }, - Spec: apiv1.QueueSpec{ - Weight: 2, + Weight: 0, }, } - var result apiv1.Queue - err = crdClient.Post(). - Resource(apiv1.QueuePlural). - Namespace(crd01.Namespace). - Body(crd01). - Do().Into(&result) - if err != nil { - return fmt.Errorf("fail to create crd crd01, %#v", err) + cases := []struct { + name string + ns string + weight int + }{ + { + name: "queue01", + ns: "ns01", + weight: 1, + }, + { + name: "queue02", + ns: "ns02", + weight: 2, + }, } - err = crdClient.Post(). - Resource(apiv1.QueuePlural). - Namespace(crd02.Namespace). - Body(crd02). - Do().Into(&result) + + for _, c := range cases { + crd.Name = c.name + crd.Namespace = c.ns + crd.Spec.Weight = c.weight + + var result apiv1.Queue + err = crdClient.Post(). + Resource(apiv1.QueuePlural). + Namespace(crd.Namespace). + Body(crd). + Do().Into(&result) + if err != nil { + return fmt.Errorf("fail to create crd %s, %#v", crd.Name, err) + } + } + + return nil +} + +// prepareCRDForPreemption create one Queue "queue03" +// "queue03" is under namespace "ns03" and has attribute "weight=2" +func prepareCRDForPreemption(config *restclient.Config) error { + extensionscs, err := apiextensionsclient.NewForConfig(config) if err != nil { - return fmt.Errorf("fail to create crd crd02, %#v", err) + return fmt.Errorf("fail to create crd config, %#v", err) + } + + _, err = client.CreateQueueCRD(extensionscs) + if err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("fail to create crd, %#v", err) + } + + crdClient, _, err := client.NewClient(config) + if err != nil { + return fmt.Errorf("fail to create crd client, %#v", err) + } + + crd := &apiv1.Queue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx", + Namespace: "xxx", + }, + Spec: apiv1.QueueSpec{ + Weight: 0, + }, + } + + cases := []struct { + name string + ns string + weight int + }{ + { + name: "queue03", + ns: "ns03", + weight: 2, + }, + } + + for _, c := range cases { + crd.Name = c.name + crd.Namespace = c.ns + crd.Spec.Weight = c.weight + + var result apiv1.Queue + err = crdClient.Post(). + Resource(apiv1.QueuePlural). + Namespace(crd.Namespace). + Body(crd). + Do().Into(&result) + if err != nil { + return fmt.Errorf("fail to create crd %s, %#v", crd.Name, err) + } + } + + return nil +} + +// preparePods create pods under ns01 and ns02 +// 5 pods under ns01 +// 8 pods under ns02 +// each pod use 1 CPU and 1 Gi Memory +func preparePods(cs *clientset.Clientset) error { + port := v1.ContainerPort{ + ContainerPort: 6379, + } + container := v1.Container{ + Name: "key-value-store", + Image: "redis", + Resources: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("1"), + "memory": resource.MustParse("1Gi"), + }, + Requests: map[v1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("1"), + "memory": resource.MustParse("1Gi"), + }, + }, + Ports: []v1.ContainerPort{port}, + } + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx", + Namespace: "xxx", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{container}, + }, + } + + cases := []struct { + name string + ns string + }{ + { + name: "ns01-pod01", + ns: "ns01", + }, + { + name: "ns01-pod02", + ns: "ns01", + }, + { + name: "ns01-pod03", + ns: "ns01", + }, + { + name: "ns01-pod04", + ns: "ns01", + }, + { + name: "ns01-pod05", + ns: "ns01", + }, + { + name: "ns02-pod01", + ns: "ns02", + }, + { + name: "ns02-pod02", + ns: "ns02", + }, + { + name: "ns02-pod03", + ns: "ns02", + }, + { + name: "ns02-pod04", + ns: "ns02", + }, + { + name: "ns02-pod05", + ns: "ns02", + }, + { + name: "ns02-pod06", + ns: "ns02", + }, + { + name: "ns02-pod07", + ns: "ns02", + }, + { + name: "ns02-pod08", + ns: "ns02", + }, + } + + for _, c := range cases { + pod.Name = c.name + pod.Namespace = c.ns + + _, err := cs.CoreV1().Pods(pod.Namespace).Create(pod) + if err != nil { + return fmt.Errorf("fail to create pod %s, %#v", pod.Name, err) + } } return nil @@ -241,4 +447,52 @@ func TestArbitrator(t *testing.T) { if v, _ := (&cpu02).AsInt64(); v != int64(10) { t.Fatalf("after scheduler, cpu is not 10 for rq02, %#v", rq02) } + + // test preemption + err = preparePods(cs) + if err != nil { + t.Fatalf("fail to prepare Pods, %#v", err) + } + // sleep to wait pods creation done + time.Sleep(10 * time.Second) + pods01, _ := cs.CoreV1().Pods("ns01").List(metav1.ListOptions{}) + if len(pods01.Items) != 5 { + t.Fatalf("running pods size is not 5 for ns01, %#v", pods01.Items) + } + pods02, _ := cs.CoreV1().Pods("ns02").List(metav1.ListOptions{}) + if len(pods02.Items) != 8 { + t.Fatalf("running pods size is not 8 for ns02, %#v", pods02.Items) + } + + // create a new queue to trigger preemption + err = prepareCRDForPreemption(config) + if err != nil { + t.Fatalf("fail to prepare CRD for preemption, %#v", err) + } + + // sleep to wait scheduler finish + time.Sleep(20 * time.Second) + rq01, _ = cs.CoreV1().ResourceQuotas("ns01").Get("rq01", metav1.GetOptions{}) + cpu01 = rq01.Spec.Hard["limits.cpu"] + if v, _ := (&cpu01).AsInt64(); v != int64(3) { + t.Fatalf("after preemption, cpu is not 3 for rq01, %#v", rq01) + } + rq02, _ = cs.CoreV1().ResourceQuotas("ns02").Get("rq02", metav1.GetOptions{}) + cpu02 = rq02.Spec.Hard["limits.cpu"] + if v, _ := (&cpu02).AsInt64(); v != int64(6) { + t.Fatalf("after preemption, cpu is not 6 for rq02, %#v", rq02) + } + rq03, _ := cs.CoreV1().ResourceQuotas("ns03").Get("rq03", metav1.GetOptions{}) + cpu03 := rq03.Spec.Hard["limits.cpu"] + if v, _ := (&cpu03).AsInt64(); v != int64(6) { + t.Fatalf("after preemption, cpu is not 6 for rq03, %#v", rq03) + } + pods01, _ = cs.CoreV1().Pods("ns01").List(metav1.ListOptions{}) + if len(pods01.Items) != 3 { + t.Fatalf("after preemption, pods size is not 3 for ns01, %#v", pods01.Items) + } + pods02, _ = cs.CoreV1().Pods("ns02").List(metav1.ListOptions{}) + if len(pods02.Items) != 6 { + t.Fatalf("after preemption, pods size is not 6 for ns02, %#v", pods02.Items) + } } diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index 3567d48e6..d8b96a295 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -600,7 +600,7 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller s.ServiceClusterIPRange.Mask = net.CIDRMask(16, 32) s.Etcd.StorageConfig = *storageConfig s.Etcd.DefaultStorageMediaType = "application/json" - s.Admission.PluginNames = strings.Split("Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", ",") + s.Admission.PluginNames = strings.Split("Initializers,NamespaceLifecycle,LimitRanger,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", ",") s.APIEnablement.RuntimeConfig.Set("api/all=true") t.Logf("Starting kube-apiserver...")