volcano-sh
/
volcano
mirror of https://github.com/volcano-sh/volcano.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #61 from jinzhejz/jinzhejz_test 

preemption integration test and code refine
This commit is contained in:
Klaus Ma 2017-10-20 10:06:44 +08:00 committed by GitHub
parent cc3bf4b07d 5161a30215
commit ed94c64b4d
2 changed files with 328 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

400
test/integration/arbitrator/arbitrator_test.go
View File

@ -39,26 +39,34 @@ import (
restclient "k8s.io/client-go/rest"
)
// prepareNamespace prepare two namespaces "ns01" and "ns02"
// prepareNamespace prepare three namespaces "ns01" "ns02" "ns03"
func prepareNamespace(cs *clientset.Clientset) error {
ns01 := &v1.Namespace{
ns := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "ns01",
},
}
ns02 := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "ns02",
Name: "xxx",
},
}
_, err := cs.CoreV1().Namespaces().Create(ns01)
if err != nil {
return fmt.Errorf("fail to create namespace ns01, %#v", err)
cases := []struct {
name string
}{
{
name: "ns01",
},
{
name: "ns02",
},
{
name: "ns03",
},
}
_, err = cs.CoreV1().Namespaces().Create(ns02)
if err != nil {
return fmt.Errorf("fail to create namespace ns02, %#v", err)
for _, c := range cases {
ns.Name = c.name
_, err := cs.CoreV1().Namespaces().Create(ns)
if err != nil {
return fmt.Errorf("fail to create namespace %s, %#v", ns.Name, err)
}
}
return nil
@ -68,16 +76,16 @@ func prepareNamespace(cs *clientset.Clientset) error {
func prepareNode(cs *clientset.Clientset) error {
node := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node01",
GenerateName: "node01",
Name: "xxx",
GenerateName: "xxx",
},
Spec: v1.NodeSpec{
ExternalID: "foo",
},
Status: v1.NodeStatus{
Capacity: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("15"),
v1.ResourceMemory: resource.MustParse("15Gi"),
v1.ResourceCPU: resource.MustParse("0"),
v1.ResourceMemory: resource.MustParse("0"),
},
Phase: v1.NodeRunning,
Conditions: []v1.NodeCondition{
@ -86,33 +94,43 @@ func prepareNode(cs *clientset.Clientset) error {
},
}
_, err := cs.CoreV1().Nodes().Create(node)
if err != nil {
return fmt.Errorf("fail to create node node01, %#v", err)
cases := []struct {
name string
resourceCPU resource.Quantity
resourceMemory resource.Quantity
}{
{
name: "node01",
resourceCPU: resource.MustParse("15"),
resourceMemory: resource.MustParse("15Gi"),
},
}
for _, c := range cases {
node.Name = c.name
node.GenerateName = c.name
node.Status.Capacity[v1.ResourceCPU] = c.resourceCPU
node.Status.Capacity[v1.ResourceMemory] = c.resourceMemory
_, err := cs.CoreV1().Nodes().Create(node)
if err != nil {
return fmt.Errorf("fail to create node %s, %#v", node.Name, err)
}
}
return nil
}
// prepareResourceQuota prepare two resource quotas "rq01" and "rq02"
// "rq01" is under namespace "ns01", "rq02" is under namespace "ns02"
// "rq01" and "rq02" will not limit cpu and memory usage at first
// prepareResourceQuota prepare three resource quotas "rq01" "rq02" "rq03"
// "rq01" is under namespace "ns01"
// "rq02" is under namespace "ns02"
// "rq03" is under namespace "ns03"
// "rq01" "rq02" "rq03" will not limit cpu and memory usage at first
func prepareResourceQuota(cs *clientset.Clientset) error {
rq01 := &v1.ResourceQuota{
rq := &v1.ResourceQuota{
ObjectMeta: metav1.ObjectMeta{
Name: "rq01",
Namespace: "ns01",
},
Spec: v1.ResourceQuotaSpec{
Hard: v1.ResourceList{
v1.ResourcePods: resource.MustParse("1000"),
},
},
}
rq02 := &v1.ResourceQuota{
ObjectMeta: metav1.ObjectMeta{
Name: "rq02",
Namespace: "ns02",
Name: "xxx",
Namespace: "xxx",
},
Spec: v1.ResourceQuotaSpec{
Hard: v1.ResourceList{
@ -121,20 +139,38 @@ func prepareResourceQuota(cs *clientset.Clientset) error {
},
}
_, err := cs.CoreV1().ResourceQuotas(rq01.Namespace).Create(rq01)
if err != nil {
return fmt.Errorf("fail to create quota rq01, %#v", err)
cases := []struct {
name string
ns string
}{
{
name: "rq01",
ns: "ns01",
},
{
name: "rq02",
ns: "ns02",
},
{
name: "rq03",
ns: "ns03",
},
}
_, err = cs.CoreV1().ResourceQuotas(rq02.Namespace).Create(rq02)
if err != nil {
return fmt.Errorf("fail to create quota rq02, %#v", err)
for _, c := range cases {
rq.Name = c.name
rq.Namespace = c.ns
_, err := cs.CoreV1().ResourceQuotas(rq.Namespace).Create(rq)
if err != nil {
return fmt.Errorf("fail to create quota %s, %#v", rq.Name, err)
}
}
return nil
}
// prepareCRD prepare customer resource definition "ResourceQuotaAllocator"
// create two ResourceQuotaAllocator, "queue01" and "queue02"
// prepareCRD prepare customer resource definition "Queue"
// create two Queues, "queue01" and "queue02"
// "queue01" is under namespace "ns01" and has attribute "weight=1"
// "queue02" is under namespace "ns02" and has attribute "weight=2"
func prepareCRD(config *restclient.Config) error {
@ -153,41 +189,211 @@ func prepareCRD(config *restclient.Config) error {
return fmt.Errorf("fail to create crd client, %#v", err)
}
crd01 := &apiv1.Queue{
crd := &apiv1.Queue{
ObjectMeta: metav1.ObjectMeta{
Name: "queue01",
Namespace: "ns01",
Name: "xxx",
Namespace: "xxx",
},
Spec: apiv1.QueueSpec{
Weight: 1,
},
}
crd02 := &apiv1.Queue{
ObjectMeta: metav1.ObjectMeta{
Name: "queue02",
Namespace: "ns02",
},
Spec: apiv1.QueueSpec{
Weight: 2,
Weight: 0,
},
}
var result apiv1.Queue
err = crdClient.Post().
Resource(apiv1.QueuePlural).
Namespace(crd01.Namespace).
Body(crd01).
Do().Into(&result)
if err != nil {
return fmt.Errorf("fail to create crd crd01, %#v", err)
cases := []struct {
name string
ns string
weight int
}{
{
name: "queue01",
ns: "ns01",
weight: 1,
},
{
name: "queue02",
ns: "ns02",
weight: 2,
},
}
err = crdClient.Post().
Resource(apiv1.QueuePlural).
Namespace(crd02.Namespace).
Body(crd02).
Do().Into(&result)
for _, c := range cases {
crd.Name = c.name
crd.Namespace = c.ns
crd.Spec.Weight = c.weight
var result apiv1.Queue
err = crdClient.Post().
Resource(apiv1.QueuePlural).
Namespace(crd.Namespace).
Body(crd).
Do().Into(&result)
if err != nil {
return fmt.Errorf("fail to create crd %s, %#v", crd.Name, err)
}
}
return nil
}
// prepareCRDForPreemption create one Queue "queue03"
// "queue03" is under namespace "ns03" and has attribute "weight=2"
func prepareCRDForPreemption(config *restclient.Config) error {
extensionscs, err := apiextensionsclient.NewForConfig(config)
if err != nil {
return fmt.Errorf("fail to create crd crd02, %#v", err)
return fmt.Errorf("fail to create crd config, %#v", err)
}
_, err = client.CreateQueueCRD(extensionscs)
if err != nil && !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("fail to create crd, %#v", err)
}
crdClient, _, err := client.NewClient(config)
if err != nil {
return fmt.Errorf("fail to create crd client, %#v", err)
}
crd := &apiv1.Queue{
ObjectMeta: metav1.ObjectMeta{
Name: "xxx",
Namespace: "xxx",
},
Spec: apiv1.QueueSpec{
Weight: 0,
},
}
cases := []struct {
name string
ns string
weight int
}{
{
name: "queue03",
ns: "ns03",
weight: 2,
},
}
for _, c := range cases {
crd.Name = c.name
crd.Namespace = c.ns
crd.Spec.Weight = c.weight
var result apiv1.Queue
err = crdClient.Post().
Resource(apiv1.QueuePlural).
Namespace(crd.Namespace).
Body(crd).
Do().Into(&result)
if err != nil {
return fmt.Errorf("fail to create crd %s, %#v", crd.Name, err)
}
}
return nil
}
// preparePods create pods under ns01 and ns02
// 5 pods under ns01
// 8 pods under ns02
// each pod use 1 CPU and 1 Gi Memory
func preparePods(cs *clientset.Clientset) error {
port := v1.ContainerPort{
ContainerPort: 6379,
}
container := v1.Container{
Name: "key-value-store",
Image: "redis",
Resources: v1.ResourceRequirements{
Limits: map[v1.ResourceName]resource.Quantity{
"cpu": resource.MustParse("1"),
"memory": resource.MustParse("1Gi"),
},
Requests: map[v1.ResourceName]resource.Quantity{
"cpu": resource.MustParse("1"),
"memory": resource.MustParse("1Gi"),
},
},
Ports: []v1.ContainerPort{port},
}
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "xxx",
Namespace: "xxx",
},
Spec: v1.PodSpec{
Containers: []v1.Container{container},
},
}
cases := []struct {
name string
ns string
}{
{
name: "ns01-pod01",
ns: "ns01",
},
{
name: "ns01-pod02",
ns: "ns01",
},
{
name: "ns01-pod03",
ns: "ns01",
},
{
name: "ns01-pod04",
ns: "ns01",
},
{
name: "ns01-pod05",
ns: "ns01",
},
{
name: "ns02-pod01",
ns: "ns02",
},
{
name: "ns02-pod02",
ns: "ns02",
},
{
name: "ns02-pod03",
ns: "ns02",
},
{
name: "ns02-pod04",
ns: "ns02",
},
{
name: "ns02-pod05",
ns: "ns02",
},
{
name: "ns02-pod06",
ns: "ns02",
},
{
name: "ns02-pod07",
ns: "ns02",
},
{
name: "ns02-pod08",
ns: "ns02",
},
}
for _, c := range cases {
pod.Name = c.name
pod.Namespace = c.ns
_, err := cs.CoreV1().Pods(pod.Namespace).Create(pod)
if err != nil {
return fmt.Errorf("fail to create pod %s, %#v", pod.Name, err)
}
}
return nil
@ -241,4 +447,52 @@ func TestArbitrator(t *testing.T) {
if v, _ := (&cpu02).AsInt64(); v != int64(10) {
t.Fatalf("after scheduler, cpu is not 10 for rq02, %#v", rq02)
}
// test preemption
err = preparePods(cs)
if err != nil {
t.Fatalf("fail to prepare Pods, %#v", err)
}
// sleep to wait pods creation done
time.Sleep(10 * time.Second)
pods01, _ := cs.CoreV1().Pods("ns01").List(metav1.ListOptions{})
if len(pods01.Items) != 5 {
t.Fatalf("running pods size is not 5 for ns01, %#v", pods01.Items)
}
pods02, _ := cs.CoreV1().Pods("ns02").List(metav1.ListOptions{})
if len(pods02.Items) != 8 {
t.Fatalf("running pods size is not 8 for ns02, %#v", pods02.Items)
}
// create a new queue to trigger preemption
err = prepareCRDForPreemption(config)
if err != nil {
t.Fatalf("fail to prepare CRD for preemption, %#v", err)
}
// sleep to wait scheduler finish
time.Sleep(20 * time.Second)
rq01, _ = cs.CoreV1().ResourceQuotas("ns01").Get("rq01", metav1.GetOptions{})
cpu01 = rq01.Spec.Hard["limits.cpu"]
if v, _ := (&cpu01).AsInt64(); v != int64(3) {
t.Fatalf("after preemption, cpu is not 3 for rq01, %#v", rq01)
}
rq02, _ = cs.CoreV1().ResourceQuotas("ns02").Get("rq02", metav1.GetOptions{})
cpu02 = rq02.Spec.Hard["limits.cpu"]
if v, _ := (&cpu02).AsInt64(); v != int64(6) {
t.Fatalf("after preemption, cpu is not 6 for rq02, %#v", rq02)
}
rq03, _ := cs.CoreV1().ResourceQuotas("ns03").Get("rq03", metav1.GetOptions{})
cpu03 := rq03.Spec.Hard["limits.cpu"]
if v, _ := (&cpu03).AsInt64(); v != int64(6) {
t.Fatalf("after preemption, cpu is not 6 for rq03, %#v", rq03)
}
pods01, _ = cs.CoreV1().Pods("ns01").List(metav1.ListOptions{})
if len(pods01.Items) != 3 {
t.Fatalf("after preemption, pods size is not 3 for ns01, %#v", pods01.Items)
}
pods02, _ = cs.CoreV1().Pods("ns02").List(metav1.ListOptions{})
if len(pods02.Items) != 6 {
t.Fatalf("after preemption, pods size is not 6 for ns02, %#v", pods02.Items)
}
}

2
test/integration/framework/master_utils.go
View File

@ -600,7 +600,7 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
s.ServiceClusterIPRange.Mask = net.CIDRMask(16, 32)
s.Etcd.StorageConfig = *storageConfig
s.Etcd.DefaultStorageMediaType = "application/json"
s.Admission.PluginNames = strings.Split("Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", ",")
s.Admission.PluginNames = strings.Split("Initializers,NamespaceLifecycle,LimitRanger,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", ",")
s.APIEnablement.RuntimeConfig.Set("api/all=true")
t.Logf("Starting kube-apiserver...")