feature: Add DRA Implementation
Signed-off-by: JesseStutler <chenzicong4@huawei.com>
This commit is contained in:
JesseStutler
parent
48f2b4fd84
commit
90d92512b4
|
|
@ -0,0 +1,49 @@
|
|||
name: E2E DRA
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
tags:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
e2e_dra:
|
||||
runs-on: ubuntu-24.04
|
||||
name: E2E about DRA
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.23.x
|
||||
|
||||
- name: Install musl
|
||||
run: |
|
||||
wget http://musl.libc.org/releases/musl-1.2.1.tar.gz
|
||||
tar -xf musl-1.2.1.tar.gz && cd musl-1.2.1
|
||||
./configure
|
||||
make && sudo make install
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
|
||||
|
||||
- name: Install dependences
|
||||
run: |
|
||||
GO111MODULE="on" go install sigs.k8s.io/kind@v0.26.0
|
||||
curl -LO https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Run E2E Tests
|
||||
run: |
|
||||
export ARTIFACTS_PATH=${{ github.workspace }}/e2e-dra-logs
|
||||
make e2e-test-dra CC=/usr/local/musl/bin/musl-gcc
|
||||
|
||||
- name: Upload e2e dra logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: volcano_e2e_dra_logs
|
||||
path: ${{ github.workspace }}/e2e-dra-logs
|
||||
3
Makefile
3
Makefile
|
|
@ -143,6 +143,9 @@ e2e-test-vcctl: vcctl images
|
|||
e2e-test-stress: images
|
||||
E2E_TYPE=STRESS ./hack/run-e2e-kind.sh
|
||||
|
||||
e2e-test-dra: images
|
||||
E2E_TYPE=DRA FEATURE_GATES="DynamicResourceAllocation=true" ./hack/run-e2e-kind.sh
|
||||
|
||||
generate-yaml: init manifests
|
||||
./hack/generate-yaml.sh TAG=${RELEASE_VER} CRD_VERSION=${CRD_VERSION}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
# How to Enable Dynamic Resource Allocation (DRA) in Volcano Scheduler
|
||||
|
||||
This document describes the steps required to enable Dynamic Resource Allocation (DRA) support in the Volcano scheduler.
|
||||
|
||||
## Prerequisites
|
||||
Before proceeding with the configuration steps, ensure your cluster meets the following prerequisites:
|
||||
|
||||
### Configure Cluster Nodes (Containerd)
|
||||
For nodes running containerd as the container runtime, you must enable the Container Device Interface (CDI) feature.
|
||||
This is crucial for containerd to properly interact with DRA drivers and inject dynamic resources into Pods.
|
||||
|
||||
Modify the containerd configuration file on each node (typically /etc/containerd/config.toml) to ensure the following setting is present:
|
||||
```toml
|
||||
# Enable CDI as described in
|
||||
# https://tags.cncf.io/container-device-interface#containerd-configuration
|
||||
[plugins."io.containerd.grpc.v1.cri"]
|
||||
enable_cdi = true
|
||||
cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"]
|
||||
```
|
||||
After modifying the configuration, restart the containerd service on each node for the changes to take effect. For example: `sudo systemctl restart containerd`
|
||||
|
||||
> If you are using other container runtimes, please refer to: [how-to-configure-cdi](https://github.com/cncf-tags/container-device-interface?tab=readme-ov-file#how-to-configure-cdi)
|
||||
|
||||
## 1. Configure Kube-apiserver
|
||||
DRA-related APIs are k8s built-in resources instead of CRD resources, and these resources are not registered by default in v1.32,
|
||||
so you need to set the startup parameters of kube-apiserver to manually register DRA-related APIs, add or ensure the following flag is present in your kube-apiserver manifest or configuration:
|
||||
```yaml
|
||||
--runtime-config=resource.k8s.io/v1beta1=true
|
||||
```
|
||||
|
||||
## 2. Install Volcano With DRA feature gates enabled
|
||||
When installing Volcano, you need to enable the DRA related feature gates, e.g., `DynamicResourceAllocation` must be enabled when you need to use DRA,
|
||||
you can also choose to enable the `DRAAdminAccess` feature gate to manage devices as your need.
|
||||
|
||||
When you are using helm to install Volcano, you can use following command to install Volcano with DRA feature gates enabled:
|
||||
```bash
|
||||
helm install volcano volcano/volcano --namespace volcano-system --create-namespace \
|
||||
--set custom.scheduler_feature_gates="DynamicResourceAllocation=true" \
|
||||
# Add other necessary Helm values for your installation
|
||||
```
|
||||
|
||||
When you directly use `kubectl apply -f` to install Volcano, you need to add or ensure the following flag is present in your volcano-scheduler manifest:
|
||||
```yaml
|
||||
--feature-gates=DynamicResourceAllocation=true
|
||||
```
|
||||
|
||||
## 3. Configure Volcano Scheduler Plugins
|
||||
After installing Volcano, you need to configure the Volcano scheduler's plugin configuration to enable the DRA plugin within the predicates plugin arguments.
|
||||
|
||||
Locate your Volcano scheduler configuration (A ConfigMap contains the configuration). Find the predicates plugin configuration and add or modify its arguments to enable DRA plugin.
|
||||
|
||||
An example snippet of the scheduler configuration (within the volcano-scheduler.conf key of the ConfigMap) might look like this:
|
||||
```yaml
|
||||
actions: "enqueue, allocate, backfill"
|
||||
tiers:
|
||||
- plugins:
|
||||
- name: priority
|
||||
- name: gang
|
||||
- plugins:
|
||||
- name: drf
|
||||
- name: predicates
|
||||
arguments:
|
||||
predicate.DynamicResourceAllocationEnable: true
|
||||
- name: proportion
|
||||
- name: nodeorder
|
||||
- name: binpack
|
||||
```
|
||||
|
||||
## 4. Deploy a DRA Driver
|
||||
To utilize Dynamic Resource Allocation, you need to deploy a DRA driver in your cluster. The driver is responsible for managing the lifecycle of dynamic resources.
|
||||
For example, you can refer to the [kubernetes-sigs/dra-example-driver](https://github.com/kubernetes-sigs/dra-example-driver) to deploy a example DRA driver for testing.
|
||||
|
||||
For some DRA Drivers which have already been used in actual production, you can refer to:
|
||||
- [NVIDIA/k8s-dra-driver-gpu](https://github.com/NVIDIA/k8s-dra-driver-gpu)
|
||||
- [intel/intel-resource-drivers-for-kubernetes](https://github.com/intel/intel-resource-drivers-for-kubernetes)
|
||||
|
||||
3
go.mod
3
go.mod
|
|
@ -54,9 +54,11 @@ require (
|
|||
|
||||
require (
|
||||
cel.dev/expr v0.18.0 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
|
||||
github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
|
||||
github.com/container-storage-interface/spec v1.9.0 // indirect
|
||||
github.com/containerd/containerd/api v1.7.19 // indirect
|
||||
github.com/containerd/errdefs v0.1.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
|
|
@ -72,6 +74,7 @@ require (
|
|||
github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect
|
||||
github.com/moby/spdystream v0.5.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.0 // indirect
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
|
||||
github.com/opencontainers/runtime-spec v1.2.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -2,6 +2,8 @@ cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo=
|
|||
cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI=
|
||||
github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
|
|
@ -29,6 +31,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
|
|||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok=
|
||||
github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE=
|
||||
github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY=
|
||||
github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0=
|
||||
github.com/containerd/containerd/api v1.7.19 h1:VWbJL+8Ap4Ju2mx9c9qS1uFSB1OVYr5JJrW2yT5vFoA=
|
||||
github.com/containerd/containerd/api v1.7.19/go.mod h1:fwGavl3LNwAV5ilJ0sbrABL44AQxmNjDRcwheXDb6Ig=
|
||||
github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM=
|
||||
|
|
@ -49,6 +53,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8
|
|||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
|
||||
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||
github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8=
|
||||
github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -203,6 +209,8 @@ github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9Kou
|
|||
github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4=
|
||||
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
|
||||
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
|
|
@ -387,6 +395,7 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
|
|||
|
|
@ -1,15 +1,29 @@
|
|||
# this config file contains all config fields with comments
|
||||
kind: Cluster
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
featureGates:
|
||||
DynamicResourceAllocation: true
|
||||
DRAResourceClaimDeviceStatus: true
|
||||
containerdConfigPatches:
|
||||
# Enable CDI as described in
|
||||
# https://tags.cncf.io/container-device-interface#containerd-configuration
|
||||
- |-
|
||||
[plugins."io.containerd.grpc.v1.cri"]
|
||||
enable_cdi = true
|
||||
# 1 control plane node and 4 workers
|
||||
nodes:
|
||||
# the control plane node config
|
||||
- role: control-plane
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
apiVersion: kubelet.config.k8s.io/v1beta1
|
||||
kind: KubeletConfiguration
|
||||
containerLogMaxSize: "50Mi"
|
||||
- |
|
||||
apiVersion: kubelet.config.k8s.io/v1beta1
|
||||
kind: KubeletConfiguration
|
||||
containerLogMaxSize: "50Mi"
|
||||
- |
|
||||
kind: ClusterConfiguration
|
||||
apiServer:
|
||||
extraArgs:
|
||||
runtime-config: "resource.k8s.io/v1beta1=true"
|
||||
# the four workers
|
||||
- role: worker
|
||||
- role: worker
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ custom:
|
|||
effect: "NoSchedule"
|
||||
default_ns:
|
||||
node-role.kubernetes.io/control-plane: ""
|
||||
scheduler_feature_gates: ${FEATURE_GATES}
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -151,6 +152,7 @@ case ${E2E_TYPE} in
|
|||
KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/schedulingbase/
|
||||
KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/schedulingaction/
|
||||
KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/vcctl/
|
||||
KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress --focus="DRA E2E Test" ./test/e2e/dra/
|
||||
;;
|
||||
"JOBP")
|
||||
echo "Running parallel job e2e suite..."
|
||||
|
|
@ -176,6 +178,10 @@ case ${E2E_TYPE} in
|
|||
echo "Running stress e2e suite..."
|
||||
KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/stress/
|
||||
;;
|
||||
"DRA")
|
||||
echo "Running dra e2e suite..."
|
||||
KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress --focus="DRA E2E Test" ./test/e2e/dra/
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ $? -ne 0 ]]; then
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ tiers:
|
|||
- name: drf
|
||||
enablePreemptable: false
|
||||
- name: predicates
|
||||
arguments:
|
||||
predicate.DynamicResourceAllocationEnable: true
|
||||
- name: proportion
|
||||
- name: nodeorder
|
||||
- name: binpack
|
||||
|
|
|
|||
|
|
@ -110,6 +110,15 @@ rules:
|
|||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "create", "update", "watch"]
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["resourceclaims"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["resourceclaims/status"]
|
||||
verbs: ["update"]
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["deviceclasses","resourceslices"]
|
||||
verbs: ["get", "list", "watch", "create"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
|
|
|
|||
|
|
@ -4667,6 +4667,15 @@ rules:
|
|||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "create", "update", "watch"]
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["resourceclaims"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["resourceclaims/status"]
|
||||
verbs: ["update"]
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["deviceclasses","resourceslices"]
|
||||
verbs: ["get", "list", "watch", "create"]
|
||||
---
|
||||
# Source: volcano/templates/scheduler.yaml
|
||||
kind: ClusterRoleBinding
|
||||
|
|
|
|||
|
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -0,0 +1,191 @@
|
|||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
https://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2013-2018 Docker, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -402,6 +402,12 @@ func (alloc *Action) allocateResourcesForTasks(tasks *util.PriorityQueue, job *a
|
|||
}
|
||||
|
||||
if len(predicateNodes) == 0 {
|
||||
// TODO: Need to add PostFilter extension point implementation here. For example, the DRA plugin includes the PostFilter extension point,
|
||||
// but the DRA's PostFilter only occurs in extreme error conditions: Suppose a pod uses two claims. In the first scheduling attempt,
|
||||
// a node is picked and PreBind manages to update the first claim so that it is allocated and reserved for the pod.
|
||||
// But then updating the second claim fails (e.g., apiserver down) and the scheduler has to retry. During the next pod scheduling attempt,
|
||||
// the original node is no longer usable for other reasons. Other nodes are not usable either because of the allocated claim.
|
||||
// The DRA scheduler plugin detects that and then when scheduling fails (= no node passed filtering), it recovers by de-allocating the allocated claim in PostFilter.
|
||||
job.NodesFitErrors[task.UID] = fitErrors
|
||||
// Assume that all left tasks are allocatable, but can not meet gang-scheduling min member,
|
||||
// so we should break from continuously allocating.
|
||||
|
|
|
|||
|
|
@ -23,9 +23,14 @@ import (
|
|||
|
||||
"github.com/stretchr/testify/assert"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
resourcev1beta1 "k8s.io/api/resource/v1beta1"
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/utils/ptr"
|
||||
|
||||
"volcano.sh/apis/pkg/apis/scheduling"
|
||||
schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
|
||||
|
|
@ -1753,3 +1758,134 @@ func TestAllocateWithPVC(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocateWithDRA(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DynamicResourceAllocation, true)
|
||||
|
||||
plugins := map[string]framework.PluginBuilder{
|
||||
gang.PluginName: gang.New,
|
||||
predicates.PluginName: predicates.New,
|
||||
}
|
||||
|
||||
options.ServerOpts = &options.ServerOption{
|
||||
MinNodesToFind: 100,
|
||||
MinPercentageOfNodesToFind: 5,
|
||||
PercentageOfNodesToFind: 100,
|
||||
}
|
||||
|
||||
trueValue := true
|
||||
tiers := []conf.Tier{
|
||||
{
|
||||
Plugins: []conf.PluginOption{
|
||||
{
|
||||
Name: gang.PluginName,
|
||||
EnabledJobReady: &trueValue,
|
||||
EnabledPredicate: &trueValue,
|
||||
EnabledJobPipelined: &trueValue,
|
||||
EnabledTaskOrder: &trueValue,
|
||||
},
|
||||
{
|
||||
Name: predicates.PluginName,
|
||||
EnabledPredicate: &trueValue,
|
||||
Arguments: framework.Arguments{
|
||||
predicates.DynamicResourceAllocationEnable: trueValue,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
tests := []uthelper.TestCommonStruct{
|
||||
{
|
||||
Name: "Allocate normal resourceClaim successfully",
|
||||
ResourceClaims: []*resourcev1beta1.ResourceClaim{
|
||||
util.BuildResourceClaim("c1", "claim1",
|
||||
[]resourcev1beta1.DeviceRequest{util.BuildDeviceRequest("gpu", "gpu.example.com", nil, nil, nil)},
|
||||
nil, nil),
|
||||
},
|
||||
ResourceSlices: []*resourcev1beta1.ResourceSlice{
|
||||
util.BuildResourceSlice("n1-slice1", "gpu.example.com", "n1", resourcev1beta1.ResourcePool{Name: "gpu-worker", Generation: 1, ResourceSliceCount: 1},
|
||||
[]resourcev1beta1.Device{
|
||||
util.BuildDevice("gpu-1", nil, nil),
|
||||
}),
|
||||
},
|
||||
DeviceClasses: []*resourcev1beta1.DeviceClass{
|
||||
util.BuildDeviceClass("gpu.example.com", []resourcev1beta1.DeviceSelector{
|
||||
{CEL: &resourcev1beta1.CELDeviceSelector{
|
||||
Expression: fmt.Sprintf(`device.driver == 'gpu.example.com'`),
|
||||
}},
|
||||
}, nil),
|
||||
},
|
||||
PodGroups: []*schedulingv1.PodGroup{
|
||||
util.BuildPodGroup("pg1", "c1", "c1", 1, nil, schedulingv1.PodGroupInqueue),
|
||||
},
|
||||
Pods: []*v1.Pod{
|
||||
util.BuildPodWithResourceClaim("c1", "p1", "", v1.PodPending, api.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string),
|
||||
[]v1.ResourceClaim{{Name: "gpu"}}, []v1.PodResourceClaim{{Name: "gpu", ResourceClaimName: ptr.To("claim1")}}),
|
||||
},
|
||||
Queues: []*schedulingv1.Queue{
|
||||
util.BuildQueue("c1", 1, nil),
|
||||
},
|
||||
Nodes: []*v1.Node{
|
||||
util.BuildNode("n1", api.BuildResourceList("2", "4Gi", []api.ScalarResource{{Name: "pods", Value: "10"}}...), make(map[string]string)),
|
||||
},
|
||||
ExpectStatus: map[api.JobID]scheduling.PodGroupPhase{
|
||||
"c1/pg1": scheduling.PodGroupRunning,
|
||||
},
|
||||
ExpectBindMap: map[string]string{
|
||||
"c1/p1": "n1",
|
||||
},
|
||||
ExpectBindsNum: 1,
|
||||
},
|
||||
{
|
||||
Name: "claim cel runtime errors",
|
||||
ResourceClaims: []*resourcev1beta1.ResourceClaim{
|
||||
util.BuildResourceClaim("c1", "claim1",
|
||||
[]resourcev1beta1.DeviceRequest{util.BuildDeviceRequest("gpu", "gpu.example.com", nil, nil, nil)},
|
||||
nil, nil),
|
||||
},
|
||||
ResourceSlices: []*resourcev1beta1.ResourceSlice{
|
||||
util.BuildResourceSlice("n1-slice1", "gpu.example.com", "n1", resourcev1beta1.ResourcePool{Name: "gpu-worker", Generation: 1, ResourceSliceCount: 1},
|
||||
[]resourcev1beta1.Device{
|
||||
util.BuildDevice("gpu-1", nil, nil),
|
||||
}),
|
||||
},
|
||||
DeviceClasses: []*resourcev1beta1.DeviceClass{
|
||||
util.BuildDeviceClass("gpu.example.com", []resourcev1beta1.DeviceSelector{
|
||||
{CEL: &resourcev1beta1.CELDeviceSelector{
|
||||
Expression: fmt.Sprintf(`device.attributes["%s"].%s`, "some-driver", resourcev1beta1.QualifiedName("healthy")),
|
||||
}},
|
||||
}, nil),
|
||||
},
|
||||
PodGroups: []*schedulingv1.PodGroup{
|
||||
util.BuildPodGroup("pg1", "c1", "c1", 1, nil, schedulingv1.PodGroupInqueue),
|
||||
},
|
||||
Pods: []*v1.Pod{
|
||||
util.BuildPodWithResourceClaim("c1", "p1", "", v1.PodPending, api.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string),
|
||||
[]v1.ResourceClaim{{Name: "gpu"}}, []v1.PodResourceClaim{{Name: "gpu", ResourceClaimName: ptr.To("claim1")}}),
|
||||
},
|
||||
Queues: []*schedulingv1.Queue{
|
||||
util.BuildQueue("c1", 1, nil),
|
||||
},
|
||||
Nodes: []*v1.Node{
|
||||
util.BuildNode("n1", api.BuildResourceList("2", "4Gi", []api.ScalarResource{{Name: "pods", Value: "10"}}...), make(map[string]string)),
|
||||
},
|
||||
ExpectStatus: map[api.JobID]scheduling.PodGroupPhase{
|
||||
"c1/pg1": scheduling.PodGroupInqueue,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
test.Plugins = plugins
|
||||
test.RegisterSession(tiers, nil)
|
||||
defer test.Close()
|
||||
action := New()
|
||||
test.Run([]framework.Action{action})
|
||||
if err := test.CheckAll(i); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,7 +49,10 @@ import (
|
|||
"k8s.io/client-go/util/workqueue"
|
||||
"k8s.io/klog/v2"
|
||||
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
|
||||
kubefeatures "k8s.io/kubernetes/pkg/features"
|
||||
k8sframework "k8s.io/kubernetes/pkg/scheduler/framework"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources"
|
||||
"k8s.io/kubernetes/pkg/scheduler/util/assumecache"
|
||||
"stathat.com/c/consistent"
|
||||
|
||||
batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
|
||||
|
|
@ -65,7 +68,6 @@ import (
|
|||
topologyinformerv1alpha1 "volcano.sh/apis/pkg/client/informers/externalversions/topology/v1alpha1"
|
||||
"volcano.sh/volcano/cmd/scheduler/app/options"
|
||||
"volcano.sh/volcano/pkg/features"
|
||||
"volcano.sh/volcano/pkg/scheduler/api"
|
||||
schedulingapi "volcano.sh/volcano/pkg/scheduler/api"
|
||||
"volcano.sh/volcano/pkg/scheduler/metrics"
|
||||
"volcano.sh/volcano/pkg/scheduler/metrics/source"
|
||||
|
|
@ -171,6 +173,9 @@ type SchedulerCache struct {
|
|||
multiSchedulerInfo
|
||||
|
||||
binderRegistry *BinderRegistry
|
||||
|
||||
// sharedDRAManager is used in DRA plugin, contains resourceClaimTracker, resourceSliceLister and deviceClassLister
|
||||
sharedDRAManager k8sframework.SharedDRAManager
|
||||
}
|
||||
|
||||
type multiSchedulerInfo struct {
|
||||
|
|
@ -744,6 +749,14 @@ func (sc *SchedulerCache) addEventHandler() {
|
|||
UpdateFunc: sc.UpdateHyperNode,
|
||||
DeleteFunc: sc.DeleteHyperNode,
|
||||
})
|
||||
|
||||
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DynamicResourceAllocation) {
|
||||
ctx := context.TODO()
|
||||
logger := klog.FromContext(ctx)
|
||||
resourceClaimInformer := informerFactory.Resource().V1beta1().ResourceClaims().Informer()
|
||||
resourceClaimCache := assumecache.NewAssumeCache(logger, resourceClaimInformer, "ResourceClaim", "", nil)
|
||||
sc.sharedDRAManager = dynamicresources.NewDRAManager(ctx, resourceClaimCache, informerFactory)
|
||||
}
|
||||
}
|
||||
|
||||
// Run starts the schedulerCache
|
||||
|
|
@ -1423,6 +1436,10 @@ func (sc *SchedulerCache) Snapshot() *schedulingapi.ClusterInfo {
|
|||
return snapshot
|
||||
}
|
||||
|
||||
func (sc *SchedulerCache) SharedDRAManager() k8sframework.SharedDRAManager {
|
||||
return sc.sharedDRAManager
|
||||
}
|
||||
|
||||
// String returns information about the cache in a string format
|
||||
func (sc *SchedulerCache) String() string {
|
||||
sc.Mutex.Lock()
|
||||
|
|
@ -1539,7 +1556,7 @@ func (sc *SchedulerCache) UpdateJobStatus(job *schedulingapi.JobInfo, updatePGSt
|
|||
|
||||
func (sc *SchedulerCache) updateJobAnnotations(job *schedulingapi.JobInfo) {
|
||||
sc.Mutex.Lock()
|
||||
sc.Jobs[job.UID].PodGroup.GetAnnotations()[api.JobAllocatedHyperNode] = job.PodGroup.GetAnnotations()[api.JobAllocatedHyperNode]
|
||||
sc.Jobs[job.UID].PodGroup.GetAnnotations()[schedulingapi.JobAllocatedHyperNode] = job.PodGroup.GetAnnotations()[schedulingapi.JobAllocatedHyperNode]
|
||||
sc.Mutex.Unlock()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import (
|
|||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework"
|
||||
|
||||
vcclient "volcano.sh/apis/pkg/client/clientset/versioned"
|
||||
"volcano.sh/volcano/pkg/scheduler/api"
|
||||
|
|
@ -83,6 +84,9 @@ type Cache interface {
|
|||
|
||||
// RegisterBinder registers the passed binder to the cache's binderRegistry
|
||||
RegisterBinder(name string, binder interface{})
|
||||
|
||||
// SharedDRAManager returns the shared DRAManager
|
||||
SharedDRAManager() framework.SharedDRAManager
|
||||
}
|
||||
|
||||
// Binder interface for binding task and hostname
|
||||
|
|
|
|||
|
|
@ -757,6 +757,11 @@ func (ssn *Session) RecordPodGroupEvent(podGroup *api.PodGroup, eventType, reaso
|
|||
ssn.recorder.Eventf(pg, eventType, reason, msg)
|
||||
}
|
||||
|
||||
// SharedDRAManager returns the shared DRAManager from cache
|
||||
func (ssn *Session) SharedDRAManager() k8sframework.SharedDRAManager {
|
||||
return ssn.cache.SharedDRAManager()
|
||||
}
|
||||
|
||||
// String return nodes and jobs information in the session
|
||||
func (ssn *Session) String() string {
|
||||
msg := fmt.Sprintf("Session %v: \n", ssn.UID)
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import (
|
|||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/scheduler/apis/config"
|
||||
k8sframework "k8s.io/kubernetes/pkg/scheduler/framework"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/dynamicresources"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
|
||||
|
|
@ -78,6 +79,9 @@ const (
|
|||
// VolumeBindingEnable is the key for enabling Volume Binding Predicates in scheduler configmap
|
||||
VolumeBindingEnable = "predicate.VolumeBindingEnable"
|
||||
|
||||
// DynamicResourceAllocationEnable is the key for enabling Dynamic Resource Allocation Predicates in scheduler configmap
|
||||
DynamicResourceAllocationEnable = "predicate.DynamicResourceAllocationEnable"
|
||||
|
||||
// CachePredicate control cache predicate feature
|
||||
CachePredicate = "predicate.CacheEnable"
|
||||
|
||||
|
|
@ -99,6 +103,8 @@ type predicatesPlugin struct {
|
|||
pluginArguments framework.Arguments
|
||||
// The VolumeBindingPlugin needs to store to execute in PreBind and PreBindRollBack
|
||||
volumeBindingPlugin *vbcap.VolumeBinding
|
||||
// The DynamicResourceAllocationPlugin needs to store to execute in PreBind and PreBindRollBack
|
||||
dynamicResourceAllocationPlugin *dynamicresources.DynamicResources
|
||||
}
|
||||
|
||||
// New return predicate plugin
|
||||
|
|
@ -116,17 +122,18 @@ type baseResource struct {
|
|||
}
|
||||
|
||||
type predicateEnable struct {
|
||||
nodeAffinityEnable bool
|
||||
nodePortEnable bool
|
||||
taintTolerationEnable bool
|
||||
podAffinityEnable bool
|
||||
nodeVolumeLimitsEnable bool
|
||||
volumeZoneEnable bool
|
||||
podTopologySpreadEnable bool
|
||||
cacheEnable bool
|
||||
proportionalEnable bool
|
||||
volumeBindingEnable bool
|
||||
proportional map[v1.ResourceName]baseResource
|
||||
nodeAffinityEnable bool
|
||||
nodePortEnable bool
|
||||
taintTolerationEnable bool
|
||||
podAffinityEnable bool
|
||||
nodeVolumeLimitsEnable bool
|
||||
volumeZoneEnable bool
|
||||
podTopologySpreadEnable bool
|
||||
cacheEnable bool
|
||||
proportionalEnable bool
|
||||
volumeBindingEnable bool
|
||||
dynamicResourceAllocationEnable bool
|
||||
proportional map[v1.ResourceName]baseResource
|
||||
}
|
||||
|
||||
// bind context extension information of predicates
|
||||
|
|
@ -168,16 +175,17 @@ func enablePredicate(args framework.Arguments) predicateEnable {
|
|||
*/
|
||||
|
||||
predicate := predicateEnable{
|
||||
nodeAffinityEnable: true,
|
||||
nodePortEnable: true,
|
||||
taintTolerationEnable: true,
|
||||
podAffinityEnable: true,
|
||||
nodeVolumeLimitsEnable: true,
|
||||
volumeZoneEnable: true,
|
||||
podTopologySpreadEnable: true,
|
||||
cacheEnable: false,
|
||||
proportionalEnable: false,
|
||||
volumeBindingEnable: true,
|
||||
nodeAffinityEnable: true,
|
||||
nodePortEnable: true,
|
||||
taintTolerationEnable: true,
|
||||
podAffinityEnable: true,
|
||||
nodeVolumeLimitsEnable: true,
|
||||
volumeZoneEnable: true,
|
||||
podTopologySpreadEnable: true,
|
||||
cacheEnable: false,
|
||||
proportionalEnable: false,
|
||||
volumeBindingEnable: true,
|
||||
dynamicResourceAllocationEnable: false,
|
||||
}
|
||||
|
||||
// Checks whether predicate enable args is provided or not.
|
||||
|
|
@ -190,6 +198,7 @@ func enablePredicate(args framework.Arguments) predicateEnable {
|
|||
args.GetBool(&predicate.volumeZoneEnable, VolumeZoneEnable)
|
||||
args.GetBool(&predicate.podTopologySpreadEnable, PodTopologySpreadEnable)
|
||||
args.GetBool(&predicate.volumeBindingEnable, VolumeBindingEnable)
|
||||
args.GetBool(&predicate.dynamicResourceAllocationEnable, DynamicResourceAllocationEnable)
|
||||
|
||||
args.GetBool(&predicate.cacheEnable, CachePredicate)
|
||||
// Checks whether predicate.ProportionalEnable is provided or not, if given, modifies the value in predicateEnable struct.
|
||||
|
|
@ -323,10 +332,13 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
|
|||
EnableNodeInclusionPolicyInPodTopologySpread: utilFeature.DefaultFeatureGate.Enabled(features.NodeInclusionPolicyInPodTopologySpread),
|
||||
EnableMatchLabelKeysInPodTopologySpread: utilFeature.DefaultFeatureGate.Enabled(features.MatchLabelKeysInPodTopologySpread),
|
||||
EnableSidecarContainers: utilFeature.DefaultFeatureGate.Enabled(features.SidecarContainers),
|
||||
EnableDRAAdminAccess: utilFeature.DefaultFeatureGate.Enabled(features.DRAAdminAccess),
|
||||
EnableDynamicResourceAllocation: utilFeature.DefaultFeatureGate.Enabled(features.DynamicResourceAllocation),
|
||||
}
|
||||
// Initialize k8s plugins
|
||||
// TODO: Add more predicates, k8s.io/kubernetes/pkg/scheduler/framework/plugins/legacy_registry.go
|
||||
handle := k8s.NewFrameworkHandle(nodeMap, ssn.KubeClient(), ssn.InformerFactory())
|
||||
handle := k8s.NewFrameworkHandle(nodeMap, ssn.KubeClient(), ssn.InformerFactory(),
|
||||
k8s.WithSharedDRAManager(ssn.SharedDRAManager()))
|
||||
// 1. NodeUnschedulable
|
||||
plugin, _ := nodeunschedulable.New(context.TODO(), nil, handle, features)
|
||||
nodeUnscheduleFilter := plugin.(*nodeunschedulable.NodeUnschedulable)
|
||||
|
|
@ -377,6 +389,17 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
|
|||
|
||||
pp.volumeBindingPlugin = volumeBindingPluginInstance
|
||||
}
|
||||
// 10. DRA
|
||||
var dynamicResourceAllocationPlugin *dynamicresources.DynamicResources
|
||||
if predicate.dynamicResourceAllocationEnable {
|
||||
var err error
|
||||
plugin, err = dynamicresources.New(context.TODO(), nil, handle, features)
|
||||
if err != nil {
|
||||
klog.Fatalf("failed to create dra plugin with err: %v", err)
|
||||
}
|
||||
dynamicResourceAllocationPlugin = plugin.(*dynamicresources.DynamicResources)
|
||||
pp.dynamicResourceAllocationPlugin = dynamicResourceAllocationPlugin
|
||||
}
|
||||
|
||||
ssn.AddPrePredicateFn(pp.Name(), func(task *api.TaskInfo) error {
|
||||
// It is safe here to directly use the state to run plugins because we have already initialized the cycle state
|
||||
|
|
@ -442,6 +465,14 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
|
|||
}
|
||||
}
|
||||
|
||||
// DRA Predicate
|
||||
if predicate.dynamicResourceAllocationEnable {
|
||||
_, status := pp.dynamicResourceAllocationPlugin.PreFilter(context.TODO(), state, task.Pod)
|
||||
if err := handleSkipPrePredicatePlugin(status, state, task, pp.dynamicResourceAllocationPlugin.Name()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
|
|
@ -636,6 +667,21 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
|
|||
}
|
||||
}
|
||||
|
||||
// Check DRA
|
||||
if predicate.dynamicResourceAllocationEnable {
|
||||
isSkipDRA := handleSkipPredicatePlugin(state, pp.dynamicResourceAllocationPlugin.Name())
|
||||
if !isSkipDRA {
|
||||
status := pp.dynamicResourceAllocationPlugin.Filter(context.TODO(), state, task.Pod, nodeInfo)
|
||||
dynamicResourceAllocationStatus := api.ConvertPredicateStatus(status)
|
||||
if dynamicResourceAllocationStatus.Code != api.Success {
|
||||
predicateStatus = append(predicateStatus, dynamicResourceAllocationStatus)
|
||||
if ShouldAbort(dynamicResourceAllocationStatus) {
|
||||
return api.NewFitErrWithStatus(task, node, predicateStatus...)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(predicateStatus) > 0 {
|
||||
return api.NewFitErrWithStatus(task, node, predicateStatus...)
|
||||
}
|
||||
|
|
@ -678,6 +724,15 @@ func (pp *predicatesPlugin) runReservePlugins(ssn *framework.Session, event *fra
|
|||
return
|
||||
}
|
||||
}
|
||||
|
||||
// DRA Reserve
|
||||
if pp.dynamicResourceAllocationPlugin != nil {
|
||||
status := pp.dynamicResourceAllocationPlugin.Reserve(context.TODO(), state, event.Task.Pod, event.Task.Pod.Spec.NodeName)
|
||||
if !status.IsSuccess() {
|
||||
event.Err = status.AsError()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (pp *predicatesPlugin) runUnReservePlugins(ssn *framework.Session, event *framework.Event) {
|
||||
|
|
@ -687,6 +742,11 @@ func (pp *predicatesPlugin) runUnReservePlugins(ssn *framework.Session, event *f
|
|||
if pp.volumeBindingPlugin != nil {
|
||||
pp.volumeBindingPlugin.Unreserve(context.TODO(), state, event.Task.Pod, event.Task.Pod.Spec.NodeName)
|
||||
}
|
||||
|
||||
// DRA UnReserve
|
||||
if pp.dynamicResourceAllocationPlugin != nil {
|
||||
pp.dynamicResourceAllocationPlugin.Unreserve(context.TODO(), state, event.Task.Pod, event.Task.Pod.Spec.NodeName)
|
||||
}
|
||||
}
|
||||
|
||||
// needsPreBind judges whether the pod needs set up extension information in bind context
|
||||
|
|
@ -707,7 +767,12 @@ func (pp *predicatesPlugin) needsPreBind(task *api.TaskInfo) bool {
|
|||
}
|
||||
}
|
||||
|
||||
// 2. TODO: With resourceClaims
|
||||
// 2. With resourceClaims
|
||||
if pp.dynamicResourceAllocationPlugin != nil {
|
||||
if len(task.Pod.Spec.ResourceClaims) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
@ -727,6 +792,14 @@ func (pp *predicatesPlugin) PreBind(ctx context.Context, bindCtx *cache.BindCont
|
|||
}
|
||||
}
|
||||
|
||||
// DRA PreBind
|
||||
if pp.dynamicResourceAllocationPlugin != nil {
|
||||
status := pp.dynamicResourceAllocationPlugin.PreBind(ctx, state, bindCtx.TaskInfo.Pod, bindCtx.TaskInfo.Pod.Spec.NodeName)
|
||||
if !status.IsSuccess() {
|
||||
return status.AsError()
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -741,6 +814,11 @@ func (pp *predicatesPlugin) PreBindRollBack(ctx context.Context, bindCtx *cache.
|
|||
if pp.volumeBindingPlugin != nil {
|
||||
pp.volumeBindingPlugin.Unreserve(ctx, state, bindCtx.TaskInfo.Pod, bindCtx.TaskInfo.Pod.Spec.NodeName)
|
||||
}
|
||||
|
||||
// DRA UnReserve
|
||||
if pp.dynamicResourceAllocationPlugin != nil {
|
||||
pp.dynamicResourceAllocationPlugin.Unreserve(ctx, state, bindCtx.TaskInfo.Pod, bindCtx.TaskInfo.Pod.Spec.NodeName)
|
||||
}
|
||||
}
|
||||
|
||||
func (pp *predicatesPlugin) SetupBindContextExtension(ssn *framework.Session, bindCtx *cache.BindContext) {
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ import (
|
|||
"k8s.io/kubernetes/pkg/scheduler/apis/config"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework/parallelize"
|
||||
"k8s.io/kubernetes/pkg/scheduler/util/assumecache"
|
||||
|
||||
scheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding"
|
||||
)
|
||||
|
|
@ -37,13 +36,23 @@ import (
|
|||
// Framework is a K8S framework who mainly provides some methods
|
||||
// about snapshot and plugins such as predicates
|
||||
type Framework struct {
|
||||
snapshot framework.SharedLister
|
||||
kubeClient kubernetes.Interface
|
||||
informerFactory informers.SharedInformerFactory
|
||||
snapshot framework.SharedLister
|
||||
kubeClient kubernetes.Interface
|
||||
informerFactory informers.SharedInformerFactory
|
||||
sharedDRAManager framework.SharedDRAManager
|
||||
}
|
||||
|
||||
var _ framework.Handle = &Framework{}
|
||||
|
||||
type Option func(*Framework)
|
||||
|
||||
// WithSharedDRAManager sets the shared DRAManager for the framework
|
||||
func WithSharedDRAManager(sharedDRAManager framework.SharedDRAManager) Option {
|
||||
return func(f *Framework) {
|
||||
f.sharedDRAManager = sharedDRAManager
|
||||
}
|
||||
}
|
||||
|
||||
// SnapshotSharedLister returns the scheduler's SharedLister of the latest NodeInfo
|
||||
// snapshot. The snapshot is taken at the beginning of a scheduling cycle and remains
|
||||
// unchanged until a pod finishes "Reserve". There is no guarantee that the information
|
||||
|
|
@ -154,24 +163,26 @@ func (f *Framework) Parallelizer() parallelize.Parallelizer {
|
|||
return parallelize.NewParallelizer(16)
|
||||
}
|
||||
|
||||
func (f *Framework) ResourceClaimCache() *assumecache.AssumeCache {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Framework) Activate(logger klog.Logger, pods map[string]*v1.Pod) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (f *Framework) SharedDRAManager() framework.SharedDRAManager {
|
||||
return nil
|
||||
return f.sharedDRAManager
|
||||
}
|
||||
|
||||
// NewFrameworkHandle creates a FrameworkHandle interface, which is used by k8s plugins.
|
||||
func NewFrameworkHandle(nodeMap map[string]*framework.NodeInfo, client kubernetes.Interface, informerFactory informers.SharedInformerFactory) framework.Handle {
|
||||
func NewFrameworkHandle(nodeMap map[string]*framework.NodeInfo, client kubernetes.Interface, informerFactory informers.SharedInformerFactory, opts ...Option) framework.Handle {
|
||||
snapshot := NewSnapshot(nodeMap)
|
||||
return &Framework{
|
||||
fw := &Framework{
|
||||
snapshot: snapshot,
|
||||
kubeClient: client,
|
||||
informerFactory: informerFactory,
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(fw)
|
||||
}
|
||||
|
||||
return fw
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import (
|
|||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
resourcev1beta1 "k8s.io/api/resource/v1beta1"
|
||||
schedulingv1 "k8s.io/api/scheduling/v1"
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
"k8s.io/apimachinery/pkg/api/equality"
|
||||
|
|
@ -70,7 +71,10 @@ type TestCommonStruct struct {
|
|||
PVs []*v1.PersistentVolume
|
||||
PVCs []*v1.PersistentVolumeClaim
|
||||
SCs []*storagev1.StorageClass
|
||||
|
||||
// DRA related resources
|
||||
ResourceSlices []*resourcev1beta1.ResourceSlice
|
||||
DeviceClasses []*resourcev1beta1.DeviceClass
|
||||
ResourceClaims []*resourcev1beta1.ResourceClaim
|
||||
// ExpectBindMap the expected bind results.
|
||||
// bind results: ns/podName -> nodeName
|
||||
ExpectBindMap map[string]string
|
||||
|
|
@ -130,6 +134,15 @@ func (test *TestCommonStruct) createSchedulerCache() *cache.SchedulerCache {
|
|||
for _, pvc := range test.PVCs {
|
||||
kubeClient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.Background(), pvc, metav1.CreateOptions{})
|
||||
}
|
||||
for _, dc := range test.DeviceClasses {
|
||||
kubeClient.ResourceV1beta1().DeviceClasses().Create(context.Background(), dc, metav1.CreateOptions{})
|
||||
}
|
||||
for _, rc := range test.ResourceClaims {
|
||||
kubeClient.ResourceV1beta1().ResourceClaims(rc.Namespace).Create(context.Background(), rc, metav1.CreateOptions{})
|
||||
}
|
||||
for _, rs := range test.ResourceSlices {
|
||||
kubeClient.ResourceV1beta1().ResourceSlices().Create(context.Background(), rs, metav1.CreateOptions{})
|
||||
}
|
||||
// need to immediately run the cache to make sure the resources are added
|
||||
schedulerCache.Run(test.stop)
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import (
|
|||
"sync"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
resourcev1beta1 "k8s.io/api/resource/v1beta1"
|
||||
schedulingv1 "k8s.io/api/scheduling/v1"
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
|
@ -87,6 +88,16 @@ func BuildPod(namespace, name, nodeName string, p v1.PodPhase, req v1.ResourceLi
|
|||
}
|
||||
}
|
||||
|
||||
// BuildPodWithResourceClaim builds a pod object with resource claim, currently the pod only contains one container
|
||||
func BuildPodWithResourceClaim(ns, name, nodeName string, p v1.PodPhase, req v1.ResourceList, groupName string, labels map[string]string, selector map[string]string,
|
||||
claimReq []v1.ResourceClaim, resourceClaims []v1.PodResourceClaim) *v1.Pod {
|
||||
pod := BuildPod(ns, name, nodeName, p, req, groupName, labels, selector)
|
||||
pod.Spec.ResourceClaims = resourceClaims
|
||||
pod.Spec.Containers[0].Resources.Claims = claimReq
|
||||
|
||||
return pod
|
||||
}
|
||||
|
||||
// BuildPodWithPVC builts Pod object with pvc volume
|
||||
func BuildPodWithPVC(namespace, name, nodename string, p v1.PodPhase, req v1.ResourceList, pvc *v1.PersistentVolumeClaim, groupName string, labels map[string]string, selector map[string]string) *v1.Pod {
|
||||
return &v1.Pod{
|
||||
|
|
@ -166,6 +177,98 @@ func BuildPV(name, scName string, capacity v1.ResourceList) *v1.PersistentVolume
|
|||
}
|
||||
}
|
||||
|
||||
func BuildDeviceRequest(name, deviceClassName string, selectors []resourcev1beta1.DeviceSelector,
|
||||
allocationMode *resourcev1beta1.DeviceAllocationMode, count *int64) resourcev1beta1.DeviceRequest {
|
||||
deviceRequest := resourcev1beta1.DeviceRequest{
|
||||
Name: name,
|
||||
DeviceClassName: deviceClassName,
|
||||
AllocationMode: resourcev1beta1.DeviceAllocationModeExactCount,
|
||||
Count: 1,
|
||||
}
|
||||
|
||||
if selectors != nil {
|
||||
deviceRequest.Selectors = selectors
|
||||
}
|
||||
|
||||
if allocationMode != nil {
|
||||
deviceRequest.AllocationMode = *allocationMode
|
||||
}
|
||||
|
||||
if allocationMode != nil && *allocationMode == resourcev1beta1.DeviceAllocationModeExactCount && count != nil {
|
||||
deviceRequest.Count = *count
|
||||
}
|
||||
|
||||
return deviceRequest
|
||||
}
|
||||
|
||||
func BuildResourceClaim(namespace, name string, deviceRequests []resourcev1beta1.DeviceRequest,
|
||||
constraints []resourcev1beta1.DeviceConstraint, config []resourcev1beta1.DeviceClaimConfiguration) *resourcev1beta1.ResourceClaim {
|
||||
rc := &resourcev1beta1.ResourceClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: namespace,
|
||||
Name: name,
|
||||
ResourceVersion: "1",
|
||||
},
|
||||
Spec: resourcev1beta1.ResourceClaimSpec{
|
||||
Devices: resourcev1beta1.DeviceClaim{
|
||||
Requests: deviceRequests,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if constraints != nil {
|
||||
rc.Spec.Devices.Constraints = constraints
|
||||
}
|
||||
|
||||
if config != nil {
|
||||
rc.Spec.Devices.Config = config
|
||||
}
|
||||
|
||||
return rc
|
||||
}
|
||||
|
||||
func BuildDeviceClass(name string, selectors []resourcev1beta1.DeviceSelector, config []resourcev1beta1.DeviceClassConfiguration) *resourcev1beta1.DeviceClass {
|
||||
dc := &resourcev1beta1.DeviceClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Spec: resourcev1beta1.DeviceClassSpec{
|
||||
Selectors: selectors,
|
||||
},
|
||||
}
|
||||
|
||||
if config != nil {
|
||||
dc.Spec.Config = config
|
||||
}
|
||||
|
||||
return dc
|
||||
}
|
||||
|
||||
func BuildDevice(name string, attributes map[resourcev1beta1.QualifiedName]resourcev1beta1.DeviceAttribute,
|
||||
capacity map[resourcev1beta1.QualifiedName]resourcev1beta1.DeviceCapacity) resourcev1beta1.Device {
|
||||
return resourcev1beta1.Device{
|
||||
Name: name,
|
||||
Basic: &resourcev1beta1.BasicDevice{
|
||||
Attributes: attributes,
|
||||
Capacity: capacity,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func BuildResourceSlice(name, driver, nodeName string, pool resourcev1beta1.ResourcePool, devices []resourcev1beta1.Device) *resourcev1beta1.ResourceSlice {
|
||||
return &resourcev1beta1.ResourceSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Spec: resourcev1beta1.ResourceSliceSpec{
|
||||
NodeName: nodeName,
|
||||
Driver: driver,
|
||||
Pool: pool,
|
||||
Devices: devices,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// BuildStorageClass build a storageclass object with specified provisioner and volumeBindingMode
|
||||
func BuildStorageClass(name, provisioner string, volumeBindingMode storagev1.VolumeBindingMode) *storagev1.StorageClass {
|
||||
return &storagev1.StorageClass{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,394 @@
|
|||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package dra
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/gomega"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
resourceapi "k8s.io/api/resource/v1beta1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/klog/v2"
|
||||
e2edra "k8s.io/kubernetes/test/e2e/dra"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
"k8s.io/utils/ptr"
|
||||
)
|
||||
|
||||
// This file is copied from kubernetes test/e2e/dra/dra.go, and change the schedulerName of the pod to volcano.
|
||||
const (
|
||||
schedulerName = "volcano"
|
||||
)
|
||||
|
||||
// builder contains a running counter to make objects unique within thir
|
||||
// namespace.
|
||||
type builder struct {
|
||||
f *framework.Framework
|
||||
driver *e2edra.Driver
|
||||
|
||||
podCounter int
|
||||
claimCounter int
|
||||
classParameters string // JSON
|
||||
}
|
||||
|
||||
// className returns the default device class name.
|
||||
func (b *builder) className() string {
|
||||
return b.f.UniqueName + b.driver.NameSuffix + "-class"
|
||||
}
|
||||
|
||||
// class returns the device class that the builder's other objects
|
||||
// reference.
|
||||
func (b *builder) class() *resourceapi.DeviceClass {
|
||||
class := &resourceapi.DeviceClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: b.className(),
|
||||
},
|
||||
}
|
||||
class.Spec.Selectors = []resourceapi.DeviceSelector{{
|
||||
CEL: &resourceapi.CELDeviceSelector{
|
||||
Expression: fmt.Sprintf(`device.driver == "%s"`, b.driver.Name),
|
||||
},
|
||||
}}
|
||||
if b.classParameters != "" {
|
||||
class.Spec.Config = []resourceapi.DeviceClassConfiguration{{
|
||||
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
||||
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
||||
Driver: b.driver.Name,
|
||||
Parameters: runtime.RawExtension{Raw: []byte(b.classParameters)},
|
||||
},
|
||||
},
|
||||
}}
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// externalClaim returns external resource claim
|
||||
// that test pods can reference
|
||||
func (b *builder) externalClaim() *resourceapi.ResourceClaim {
|
||||
b.claimCounter++
|
||||
name := "external-claim" + b.driver.NameSuffix // This is what podExternal expects.
|
||||
if b.claimCounter > 1 {
|
||||
name += fmt.Sprintf("-%d", b.claimCounter)
|
||||
}
|
||||
return &resourceapi.ResourceClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Spec: b.claimSpec(),
|
||||
}
|
||||
}
|
||||
|
||||
// claimSpec returns the device request for a claim or claim template
|
||||
// with the associated config
|
||||
func (b *builder) claimSpec() resourceapi.ResourceClaimSpec {
|
||||
parameters, _ := b.parametersEnv()
|
||||
spec := resourceapi.ResourceClaimSpec{
|
||||
Devices: resourceapi.DeviceClaim{
|
||||
Requests: []resourceapi.DeviceRequest{{
|
||||
Name: "my-request",
|
||||
DeviceClassName: b.className(),
|
||||
}},
|
||||
Config: []resourceapi.DeviceClaimConfiguration{{
|
||||
DeviceConfiguration: resourceapi.DeviceConfiguration{
|
||||
Opaque: &resourceapi.OpaqueDeviceConfiguration{
|
||||
Driver: b.driver.Name,
|
||||
Parameters: runtime.RawExtension{
|
||||
Raw: []byte(parameters),
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
return spec
|
||||
}
|
||||
|
||||
// parametersEnv returns the default user env variables as JSON (config) and key/value list (pod env).
|
||||
func (b *builder) parametersEnv() (string, []string) {
|
||||
return `{"a":"b"}`,
|
||||
[]string{"user_a", "b"}
|
||||
}
|
||||
|
||||
// makePod returns a simple pod with no resource claims.
|
||||
// The pod prints its env and waits.
|
||||
func (b *builder) pod() *v1.Pod {
|
||||
pod := e2epod.MakePod(b.f.Namespace.Name, nil, nil, b.f.NamespacePodSecurityLevel, "env && sleep 100000")
|
||||
pod.Labels = make(map[string]string)
|
||||
pod.Spec.RestartPolicy = v1.RestartPolicyNever
|
||||
// Let kubelet kill the pods quickly. Setting
|
||||
// TerminationGracePeriodSeconds to zero would bypass kubelet
|
||||
// completely because then the apiserver enables a force-delete even
|
||||
// when DeleteOptions for the pod don't ask for it (see
|
||||
// https://github.com/kubernetes/kubernetes/blob/0f582f7c3f504e807550310d00f130cb5c18c0c3/pkg/registry/core/pod/strategy.go#L151-L171).
|
||||
//
|
||||
// We don't do that because it breaks tracking of claim usage: the
|
||||
// kube-controller-manager assumes that kubelet is done with the pod
|
||||
// once it got removed or has a grace period of 0. Setting the grace
|
||||
// period to zero directly in DeletionOptions or indirectly through
|
||||
// TerminationGracePeriodSeconds causes the controller to remove
|
||||
// the pod from ReservedFor before it actually has stopped on
|
||||
// the node.
|
||||
one := int64(1)
|
||||
pod.Spec.TerminationGracePeriodSeconds = &one
|
||||
pod.ObjectMeta.GenerateName = ""
|
||||
b.podCounter++
|
||||
pod.ObjectMeta.Name = fmt.Sprintf("tester%s-%d", b.driver.NameSuffix, b.podCounter)
|
||||
pod.Spec.SchedulerName = schedulerName
|
||||
return pod
|
||||
}
|
||||
|
||||
// makePodInline adds an inline resource claim with default class name and parameters.
|
||||
func (b *builder) podInline() (*v1.Pod, *resourceapi.ResourceClaimTemplate) {
|
||||
pod := b.pod()
|
||||
pod.Spec.Containers[0].Name = "with-resource"
|
||||
podClaimName := "my-inline-claim"
|
||||
pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: podClaimName}}
|
||||
pod.Spec.ResourceClaims = []v1.PodResourceClaim{
|
||||
{
|
||||
Name: podClaimName,
|
||||
ResourceClaimTemplateName: ptr.To(pod.Name),
|
||||
},
|
||||
}
|
||||
template := &resourceapi.ResourceClaimTemplate{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: pod.Name,
|
||||
Namespace: pod.Namespace,
|
||||
},
|
||||
Spec: resourceapi.ResourceClaimTemplateSpec{
|
||||
Spec: b.claimSpec(),
|
||||
},
|
||||
}
|
||||
return pod, template
|
||||
}
|
||||
|
||||
// podInlineMultiple returns a pod with inline resource claim referenced by 3 containers
|
||||
func (b *builder) podInlineMultiple() (*v1.Pod, *resourceapi.ResourceClaimTemplate) {
|
||||
pod, template := b.podInline()
|
||||
pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy(), *pod.Spec.Containers[0].DeepCopy())
|
||||
pod.Spec.Containers[1].Name = pod.Spec.Containers[1].Name + "-1"
|
||||
pod.Spec.Containers[2].Name = pod.Spec.Containers[1].Name + "-2"
|
||||
return pod, template
|
||||
}
|
||||
|
||||
// podExternal adds a pod that references external resource claim with default class name and parameters.
|
||||
func (b *builder) podExternal() *v1.Pod {
|
||||
pod := b.pod()
|
||||
pod.Spec.Containers[0].Name = "with-resource"
|
||||
podClaimName := "resource-claim"
|
||||
externalClaimName := "external-claim" + b.driver.NameSuffix
|
||||
pod.Spec.ResourceClaims = []v1.PodResourceClaim{
|
||||
{
|
||||
Name: podClaimName,
|
||||
ResourceClaimName: &externalClaimName,
|
||||
},
|
||||
}
|
||||
pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: podClaimName}}
|
||||
return pod
|
||||
}
|
||||
|
||||
// podShared returns a pod with 3 containers that reference external resource claim with default class name and parameters.
|
||||
func (b *builder) podExternalMultiple() *v1.Pod {
|
||||
pod := b.podExternal()
|
||||
pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy(), *pod.Spec.Containers[0].DeepCopy())
|
||||
pod.Spec.Containers[1].Name = pod.Spec.Containers[1].Name + "-1"
|
||||
pod.Spec.Containers[2].Name = pod.Spec.Containers[1].Name + "-2"
|
||||
return pod
|
||||
}
|
||||
|
||||
// create takes a bunch of objects and calls their Create function.
|
||||
func (b *builder) create(ctx context.Context, objs ...klog.KMetadata) []klog.KMetadata {
|
||||
var createdObjs []klog.KMetadata
|
||||
for _, obj := range objs {
|
||||
ginkgo.By(fmt.Sprintf("creating %T %s", obj, obj.GetName()))
|
||||
var err error
|
||||
var createdObj klog.KMetadata
|
||||
switch obj := obj.(type) {
|
||||
case *resourceapi.DeviceClass:
|
||||
createdObj, err = b.f.ClientSet.ResourceV1beta1().DeviceClasses().Create(ctx, obj, metav1.CreateOptions{})
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) {
|
||||
err := b.f.ClientSet.ResourceV1beta1().DeviceClasses().Delete(ctx, createdObj.GetName(), metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err, "delete device class")
|
||||
})
|
||||
case *v1.Pod:
|
||||
createdObj, err = b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
|
||||
case *v1.ConfigMap:
|
||||
createdObj, err = b.f.ClientSet.CoreV1().ConfigMaps(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
|
||||
case *resourceapi.ResourceClaim:
|
||||
createdObj, err = b.f.ClientSet.ResourceV1beta1().ResourceClaims(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
|
||||
case *resourceapi.ResourceClaimTemplate:
|
||||
createdObj, err = b.f.ClientSet.ResourceV1beta1().ResourceClaimTemplates(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
|
||||
case *resourceapi.ResourceSlice:
|
||||
createdObj, err = b.f.ClientSet.ResourceV1beta1().ResourceSlices().Create(ctx, obj, metav1.CreateOptions{})
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) {
|
||||
err := b.f.ClientSet.ResourceV1beta1().ResourceSlices().Delete(ctx, createdObj.GetName(), metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err, "delete node resource slice")
|
||||
})
|
||||
case *appsv1.DaemonSet:
|
||||
createdObj, err = b.f.ClientSet.AppsV1().DaemonSets(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
|
||||
// Cleanup not really needed, but speeds up namespace shutdown.
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) {
|
||||
err := b.f.ClientSet.AppsV1().DaemonSets(b.f.Namespace.Name).Delete(ctx, obj.Name, metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err, "delete daemonset")
|
||||
})
|
||||
default:
|
||||
framework.Fail(fmt.Sprintf("internal error, unsupported type %T", obj), 1)
|
||||
}
|
||||
framework.ExpectNoErrorWithOffset(1, err, "create %T", obj)
|
||||
createdObjs = append(createdObjs, createdObj)
|
||||
}
|
||||
return createdObjs
|
||||
}
|
||||
|
||||
// testPod runs pod and checks if container logs contain expected environment variables
|
||||
func (b *builder) testPod(ctx context.Context, clientSet kubernetes.Interface, pod *v1.Pod, env ...string) {
|
||||
ginkgo.GinkgoHelper()
|
||||
err := e2epod.WaitForPodRunningInNamespace(ctx, clientSet, pod)
|
||||
framework.ExpectNoError(err, "start pod")
|
||||
|
||||
if len(env) == 0 {
|
||||
_, env = b.parametersEnv()
|
||||
}
|
||||
for _, container := range pod.Spec.Containers {
|
||||
testContainerEnv(ctx, clientSet, pod, container.Name, false, env...)
|
||||
}
|
||||
}
|
||||
|
||||
// envLineRE matches env output with variables set by test/e2e/dra/test-driver.
|
||||
var envLineRE = regexp.MustCompile(`^(?:admin|user|claim)_[a-zA-Z0-9_]*=.*$`)
|
||||
|
||||
func testContainerEnv(ctx context.Context, clientSet kubernetes.Interface, pod *v1.Pod, containerName string, fullMatch bool, env ...string) {
|
||||
ginkgo.GinkgoHelper()
|
||||
log, err := e2epod.GetPodLogs(ctx, clientSet, pod.Namespace, pod.Name, containerName)
|
||||
framework.ExpectNoError(err, fmt.Sprintf("get logs for container %s", containerName))
|
||||
if fullMatch {
|
||||
// Find all env variables set by the test driver.
|
||||
var actualEnv, expectEnv []string
|
||||
for _, line := range strings.Split(log, "\n") {
|
||||
if envLineRE.MatchString(line) {
|
||||
actualEnv = append(actualEnv, line)
|
||||
}
|
||||
}
|
||||
for i := 0; i < len(env); i += 2 {
|
||||
expectEnv = append(expectEnv, env[i]+"="+env[i+1])
|
||||
}
|
||||
sort.Strings(actualEnv)
|
||||
sort.Strings(expectEnv)
|
||||
gomega.Expect(actualEnv).To(gomega.Equal(expectEnv), fmt.Sprintf("container %s log output:\n%s", containerName, log))
|
||||
} else {
|
||||
for i := 0; i < len(env); i += 2 {
|
||||
envStr := fmt.Sprintf("\n%s=%s\n", env[i], env[i+1])
|
||||
gomega.Expect(log).To(gomega.ContainSubstring(envStr), fmt.Sprintf("container %s env variables", containerName))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newBuilder(f *framework.Framework, driver *e2edra.Driver) *builder {
|
||||
b := &builder{f: f, driver: driver}
|
||||
ginkgo.BeforeEach(b.setUp)
|
||||
return b
|
||||
}
|
||||
|
||||
func newBuilderNow(ctx context.Context, f *framework.Framework, driver *e2edra.Driver) *builder {
|
||||
b := &builder{f: f, driver: driver}
|
||||
b.setUp(ctx)
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *builder) setUp(ctx context.Context) {
|
||||
b.podCounter = 0
|
||||
b.claimCounter = 0
|
||||
b.create(ctx, b.class())
|
||||
ginkgo.DeferCleanup(b.tearDown)
|
||||
}
|
||||
|
||||
func (b *builder) tearDown(ctx context.Context) {
|
||||
// Before we allow the namespace and all objects in it do be deleted by
|
||||
// the framework, we must ensure that test pods and the claims that
|
||||
// they use are deleted. Otherwise the driver might get deleted first,
|
||||
// in which case deleting the claims won't work anymore.
|
||||
ginkgo.By("delete pods and claims")
|
||||
pods, err := b.listTestPods(ctx)
|
||||
framework.ExpectNoError(err, "list pods")
|
||||
for _, pod := range pods {
|
||||
if pod.DeletionTimestamp != nil {
|
||||
continue
|
||||
}
|
||||
ginkgo.By(fmt.Sprintf("deleting %T %s", &pod, klog.KObj(&pod)))
|
||||
err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{})
|
||||
if !apierrors.IsNotFound(err) {
|
||||
framework.ExpectNoError(err, "delete pod")
|
||||
}
|
||||
}
|
||||
gomega.Eventually(func() ([]v1.Pod, error) {
|
||||
return b.listTestPods(ctx)
|
||||
}).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "remaining pods despite deletion")
|
||||
|
||||
claims, err := b.f.ClientSet.ResourceV1beta1().ResourceClaims(b.f.Namespace.Name).List(ctx, metav1.ListOptions{})
|
||||
framework.ExpectNoError(err, "get resource claims")
|
||||
for _, claim := range claims.Items {
|
||||
if claim.DeletionTimestamp != nil {
|
||||
continue
|
||||
}
|
||||
ginkgo.By(fmt.Sprintf("deleting %T %s", &claim, klog.KObj(&claim)))
|
||||
err := b.f.ClientSet.ResourceV1beta1().ResourceClaims(b.f.Namespace.Name).Delete(ctx, claim.Name, metav1.DeleteOptions{})
|
||||
if !apierrors.IsNotFound(err) {
|
||||
framework.ExpectNoError(err, "delete claim")
|
||||
}
|
||||
}
|
||||
|
||||
for host, plugin := range b.driver.Nodes {
|
||||
ginkgo.By(fmt.Sprintf("waiting for resources on %s to be unprepared", host))
|
||||
gomega.Eventually(plugin.GetPreparedResources).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host)
|
||||
}
|
||||
|
||||
ginkgo.By("waiting for claims to be deallocated and deleted")
|
||||
gomega.Eventually(func() ([]resourceapi.ResourceClaim, error) {
|
||||
claims, err := b.f.ClientSet.ResourceV1beta1().ResourceClaims(b.f.Namespace.Name).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return claims.Items, nil
|
||||
}).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "claims in the namespaces")
|
||||
}
|
||||
|
||||
func (b *builder) listTestPods(ctx context.Context) ([]v1.Pod, error) {
|
||||
pods, err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var testPods []v1.Pod
|
||||
for _, pod := range pods.Items {
|
||||
if pod.Labels["app.kubernetes.io/part-of"] == "dra-test-driver" {
|
||||
continue
|
||||
}
|
||||
testPods = append(testPods, pod)
|
||||
}
|
||||
return testPods, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
Copyright 2021 The Volcano Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package dra
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestE2E(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Volcano DRA Test Suite")
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
Copyright 2025 The Volcano Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package dra
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
"k8s.io/kubernetes/test/e2e/framework/testfiles"
|
||||
e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
handleFlags()
|
||||
framework.TestContext.CloudConfig = framework.CloudConfig{
|
||||
Provider: framework.NullProvider{},
|
||||
}
|
||||
testfiles.AddFileSource(e2etestingmanifests.GetE2ETestingManifestsFS())
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func handleFlags() {
|
||||
framework.RegisterCommonFlags(flag.CommandLine)
|
||||
framework.RegisterClusterFlags(flag.CommandLine)
|
||||
flag.Parse()
|
||||
}
|
||||
|
|
@ -0,0 +1,455 @@
|
|||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package dra
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/gomega"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
resourceapi "k8s.io/api/resource/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/klog/v2"
|
||||
e2edra "k8s.io/kubernetes/test/e2e/dra"
|
||||
"k8s.io/kubernetes/test/e2e/feature"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
)
|
||||
|
||||
// The e2e test cases in this file are copied from kubernetes test/e2e/dra/dra.go, only contains scheduling-related e2e testing.
|
||||
var _ = ginkgo.Describe("DRA E2E Test", func() {
|
||||
f := framework.NewDefaultFramework("dra")
|
||||
|
||||
// The driver containers have to run with sufficient privileges to
|
||||
// modify /var/lib/kubelet/plugins.
|
||||
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
|
||||
|
||||
// claimTests tries out several different combinations of pods with
|
||||
// claims, both inline and external.
|
||||
claimTests := func(b *builder, driver *e2edra.Driver) {
|
||||
ginkgo.It("supports simple pod referencing inline resource claim", func(ctx context.Context) {
|
||||
pod, template := b.podInline()
|
||||
b.create(ctx, pod, template)
|
||||
b.testPod(ctx, f.ClientSet, pod)
|
||||
})
|
||||
|
||||
ginkgo.It("supports inline claim referenced by multiple containers", func(ctx context.Context) {
|
||||
pod, template := b.podInlineMultiple()
|
||||
b.create(ctx, pod, template)
|
||||
b.testPod(ctx, f.ClientSet, pod)
|
||||
})
|
||||
|
||||
ginkgo.It("supports simple pod referencing external resource claim", func(ctx context.Context) {
|
||||
pod := b.podExternal()
|
||||
claim := b.externalClaim()
|
||||
b.create(ctx, claim, pod)
|
||||
b.testPod(ctx, f.ClientSet, pod)
|
||||
})
|
||||
|
||||
ginkgo.It("supports external claim referenced by multiple pods", func(ctx context.Context) {
|
||||
pod1 := b.podExternal()
|
||||
pod2 := b.podExternal()
|
||||
pod3 := b.podExternal()
|
||||
claim := b.externalClaim()
|
||||
b.create(ctx, claim, pod1, pod2, pod3)
|
||||
|
||||
for _, pod := range []*v1.Pod{pod1, pod2, pod3} {
|
||||
b.testPod(ctx, f.ClientSet, pod)
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("supports external claim referenced by multiple containers of multiple pods", func(ctx context.Context) {
|
||||
pod1 := b.podExternalMultiple()
|
||||
pod2 := b.podExternalMultiple()
|
||||
pod3 := b.podExternalMultiple()
|
||||
claim := b.externalClaim()
|
||||
b.create(ctx, claim, pod1, pod2, pod3)
|
||||
|
||||
for _, pod := range []*v1.Pod{pod1, pod2, pod3} {
|
||||
b.testPod(ctx, f.ClientSet, pod)
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.It("supports init containers", func(ctx context.Context) {
|
||||
pod, template := b.podInline()
|
||||
pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0]}
|
||||
pod.Spec.InitContainers[0].Name += "-init"
|
||||
// This must succeed for the pod to start.
|
||||
pod.Spec.InitContainers[0].Command = []string{"sh", "-c", "env | grep user_a=b"}
|
||||
b.create(ctx, pod, template)
|
||||
|
||||
b.testPod(ctx, f.ClientSet, pod)
|
||||
})
|
||||
|
||||
ginkgo.It("removes reservation from claim when pod is done", func(ctx context.Context) {
|
||||
pod := b.podExternal()
|
||||
claim := b.externalClaim()
|
||||
pod.Spec.Containers[0].Command = []string{"true"}
|
||||
b.create(ctx, claim, pod)
|
||||
|
||||
ginkgo.By("waiting for pod to finish")
|
||||
framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish")
|
||||
ginkgo.By("waiting for claim to be unreserved")
|
||||
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
||||
return f.ClientSet.ResourceV1beta1().ResourceClaims(pod.Namespace).Get(ctx, claim.Name, metav1.GetOptions{})
|
||||
}).WithTimeout(f.Timeouts.PodDelete).Should(gomega.HaveField("Status.ReservedFor", gomega.BeEmpty()), "reservation should have been removed")
|
||||
})
|
||||
|
||||
ginkgo.It("deletes generated claims when pod is done", func(ctx context.Context) {
|
||||
pod, template := b.podInline()
|
||||
pod.Spec.Containers[0].Command = []string{"true"}
|
||||
b.create(ctx, template, pod)
|
||||
|
||||
ginkgo.By("waiting for pod to finish")
|
||||
framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish")
|
||||
ginkgo.By("waiting for claim to be deleted")
|
||||
gomega.Eventually(ctx, func(ctx context.Context) ([]resourceapi.ResourceClaim, error) {
|
||||
claims, err := f.ClientSet.ResourceV1beta1().ResourceClaims(pod.Namespace).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return claims.Items, nil
|
||||
}).WithTimeout(f.Timeouts.PodDelete).Should(gomega.BeEmpty(), "claim should have been deleted")
|
||||
})
|
||||
|
||||
ginkgo.It("does not delete generated claims when pod is restarting", func(ctx context.Context) {
|
||||
pod, template := b.podInline()
|
||||
pod.Spec.Containers[0].Command = []string{"sh", "-c", "sleep 1; exit 1"}
|
||||
pod.Spec.RestartPolicy = v1.RestartPolicyAlways
|
||||
b.create(ctx, template, pod)
|
||||
|
||||
ginkgo.By("waiting for pod to restart twice")
|
||||
gomega.Eventually(ctx, func(ctx context.Context) (*v1.Pod, error) {
|
||||
return f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
|
||||
}).WithTimeout(f.Timeouts.PodStartSlow).Should(gomega.HaveField("Status.ContainerStatuses", gomega.ContainElements(gomega.HaveField("RestartCount", gomega.BeNumerically(">=", 2)))))
|
||||
})
|
||||
|
||||
ginkgo.It("must deallocate after use", func(ctx context.Context) {
|
||||
pod := b.podExternal()
|
||||
claim := b.externalClaim()
|
||||
b.create(ctx, claim, pod)
|
||||
|
||||
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
||||
return b.f.ClientSet.ResourceV1beta1().ResourceClaims(b.f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
||||
}).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil)))
|
||||
|
||||
b.testPod(ctx, f.ClientSet, pod)
|
||||
|
||||
ginkgo.By(fmt.Sprintf("deleting pod %s", klog.KObj(pod)))
|
||||
framework.ExpectNoError(b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{}))
|
||||
|
||||
ginkgo.By("waiting for claim to get deallocated")
|
||||
gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) {
|
||||
return b.f.ClientSet.ResourceV1beta1().ResourceClaims(b.f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
||||
}).WithTimeout(f.Timeouts.PodDelete).Should(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil)))
|
||||
})
|
||||
|
||||
f.It("must be possible for the driver to update the ResourceClaim.Status.Devices once allocated", feature.DRAResourceClaimDeviceStatus, func(ctx context.Context) {
|
||||
pod := b.podExternal()
|
||||
claim := b.externalClaim()
|
||||
b.create(ctx, claim, pod)
|
||||
|
||||
b.testPod(ctx, f.ClientSet, pod)
|
||||
|
||||
allocatedResourceClaim, err := b.f.ClientSet.ResourceV1beta1().ResourceClaims(b.f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(allocatedResourceClaim).ToNot(gomega.BeNil())
|
||||
gomega.Expect(allocatedResourceClaim.Status.Allocation).ToNot(gomega.BeNil())
|
||||
gomega.Expect(allocatedResourceClaim.Status.Allocation.Devices.Results).To(gomega.HaveLen(1))
|
||||
|
||||
scheduledPod, err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(scheduledPod).ToNot(gomega.BeNil())
|
||||
|
||||
gomega.Expect(allocatedResourceClaim.Status.Allocation).ToNot(gomega.BeNil())
|
||||
gomega.Expect(allocatedResourceClaim.Status.Allocation.Devices.Results).To(gomega.HaveLen(1))
|
||||
|
||||
ginkgo.By("Setting the device status a first time")
|
||||
allocatedResourceClaim.Status.Devices = append(allocatedResourceClaim.Status.Devices,
|
||||
resourceapi.AllocatedDeviceStatus{
|
||||
Driver: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Driver,
|
||||
Pool: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Pool,
|
||||
Device: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Device,
|
||||
Conditions: []metav1.Condition{{Type: "a", Status: "True", Message: "c", Reason: "d", LastTransitionTime: metav1.NewTime(time.Now().Truncate(time.Second))}},
|
||||
Data: runtime.RawExtension{Raw: []byte(`{"foo":"bar"}`)},
|
||||
NetworkData: &resourceapi.NetworkDeviceData{
|
||||
InterfaceName: "inf1",
|
||||
IPs: []string{"10.9.8.0/24", "2001:db8::/64"},
|
||||
HardwareAddress: "bc:1c:b6:3e:b8:25",
|
||||
},
|
||||
})
|
||||
// Updates the ResourceClaim from the driver on the same node as the pod.
|
||||
plugin, ok := driver.Nodes[scheduledPod.Spec.NodeName]
|
||||
if !ok {
|
||||
framework.Failf("pod got scheduled to node %s without a plugin", scheduledPod.Spec.NodeName)
|
||||
}
|
||||
updatedResourceClaim, err := plugin.UpdateStatus(ctx, allocatedResourceClaim)
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(updatedResourceClaim).ToNot(gomega.BeNil())
|
||||
gomega.Expect(updatedResourceClaim.Status.Devices).To(gomega.Equal(allocatedResourceClaim.Status.Devices))
|
||||
|
||||
ginkgo.By("Updating the device status")
|
||||
updatedResourceClaim.Status.Devices[0] = resourceapi.AllocatedDeviceStatus{
|
||||
Driver: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Driver,
|
||||
Pool: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Pool,
|
||||
Device: allocatedResourceClaim.Status.Allocation.Devices.Results[0].Device,
|
||||
Conditions: []metav1.Condition{{Type: "e", Status: "True", Message: "g", Reason: "h", LastTransitionTime: metav1.NewTime(time.Now().Truncate(time.Second))}},
|
||||
Data: runtime.RawExtension{Raw: []byte(`{"bar":"foo"}`)},
|
||||
NetworkData: &resourceapi.NetworkDeviceData{
|
||||
InterfaceName: "inf2",
|
||||
IPs: []string{"10.9.8.1/24", "2001:db8::1/64"},
|
||||
HardwareAddress: "bc:1c:b6:3e:b8:26",
|
||||
},
|
||||
}
|
||||
|
||||
updatedResourceClaim2, err := plugin.UpdateStatus(ctx, updatedResourceClaim)
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(updatedResourceClaim2).ToNot(gomega.BeNil())
|
||||
gomega.Expect(updatedResourceClaim2.Status.Devices).To(gomega.Equal(updatedResourceClaim.Status.Devices))
|
||||
|
||||
getResourceClaim, err := b.f.ClientSet.ResourceV1beta1().ResourceClaims(b.f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(getResourceClaim).ToNot(gomega.BeNil())
|
||||
gomega.Expect(getResourceClaim.Status.Devices).To(gomega.Equal(updatedResourceClaim.Status.Devices))
|
||||
})
|
||||
}
|
||||
|
||||
singleNodeTests := func() {
|
||||
nodes := e2edra.NewNodes(f, 1, 1)
|
||||
maxAllocations := 1
|
||||
numPods := 10
|
||||
generateResources := func() e2edra.Resources {
|
||||
resources := perNode(maxAllocations, nodes)()
|
||||
return resources
|
||||
}
|
||||
driver := e2edra.NewDriver(f, nodes, generateResources) // All tests get their own driver instance.
|
||||
b := newBuilder(f, driver)
|
||||
// We have to set the parameters *before* creating the class.
|
||||
b.classParameters = `{"x":"y"}`
|
||||
expectedEnv := []string{"admin_x", "y"}
|
||||
_, expected := b.parametersEnv()
|
||||
expectedEnv = append(expectedEnv, expected...)
|
||||
|
||||
ginkgo.It("supports claim and class parameters", func(ctx context.Context) {
|
||||
pod, template := b.podInline()
|
||||
b.create(ctx, pod, template)
|
||||
b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
|
||||
})
|
||||
|
||||
ginkgo.It("supports reusing resources", func(ctx context.Context) {
|
||||
var objects []klog.KMetadata
|
||||
pods := make([]*v1.Pod, numPods)
|
||||
for i := 0; i < numPods; i++ {
|
||||
pod, template := b.podInline()
|
||||
pods[i] = pod
|
||||
objects = append(objects, pod, template)
|
||||
}
|
||||
|
||||
b.create(ctx, objects...)
|
||||
|
||||
// We don't know the order. All that matters is that all of them get scheduled eventually.
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(numPods)
|
||||
for i := 0; i < numPods; i++ {
|
||||
pod := pods[i]
|
||||
go func() {
|
||||
defer ginkgo.GinkgoRecover()
|
||||
defer wg.Done()
|
||||
b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
|
||||
err := f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err, "delete pod")
|
||||
framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, time.Duration(numPods)*f.Timeouts.PodStartSlow))
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
})
|
||||
|
||||
ginkgo.It("supports sharing a claim concurrently", func(ctx context.Context) {
|
||||
var objects []klog.KMetadata
|
||||
objects = append(objects, b.externalClaim())
|
||||
pods := make([]*v1.Pod, numPods)
|
||||
for i := 0; i < numPods; i++ {
|
||||
pod := b.podExternal()
|
||||
pods[i] = pod
|
||||
objects = append(objects, pod)
|
||||
}
|
||||
|
||||
b.create(ctx, objects...)
|
||||
|
||||
// We don't know the order. All that matters is that all of them get scheduled eventually.
|
||||
f.Timeouts.PodStartSlow *= time.Duration(numPods)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(numPods)
|
||||
for i := 0; i < numPods; i++ {
|
||||
pod := pods[i]
|
||||
go func() {
|
||||
defer ginkgo.GinkgoRecover()
|
||||
defer wg.Done()
|
||||
b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
})
|
||||
|
||||
ginkgo.It("retries pod scheduling after creating device class", func(ctx context.Context) {
|
||||
var objects []klog.KMetadata
|
||||
pod, template := b.podInline()
|
||||
deviceClassName := template.Spec.Spec.Devices.Requests[0].DeviceClassName
|
||||
class, err := f.ClientSet.ResourceV1beta1().DeviceClasses().Get(ctx, deviceClassName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
deviceClassName += "-b"
|
||||
template.Spec.Spec.Devices.Requests[0].DeviceClassName = deviceClassName
|
||||
objects = append(objects, template, pod)
|
||||
b.create(ctx, objects...)
|
||||
|
||||
framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace))
|
||||
|
||||
class.UID = ""
|
||||
class.ResourceVersion = ""
|
||||
class.Name = deviceClassName
|
||||
b.create(ctx, class)
|
||||
|
||||
b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
|
||||
})
|
||||
|
||||
ginkgo.It("retries pod scheduling after updating device class", func(ctx context.Context) {
|
||||
var objects []klog.KMetadata
|
||||
pod, template := b.podInline()
|
||||
|
||||
// First modify the class so that it matches no nodes (for classic DRA) and no devices (structured parameters).
|
||||
deviceClassName := template.Spec.Spec.Devices.Requests[0].DeviceClassName
|
||||
class, err := f.ClientSet.ResourceV1beta1().DeviceClasses().Get(ctx, deviceClassName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
originalClass := class.DeepCopy()
|
||||
class.Spec.Selectors = []resourceapi.DeviceSelector{{
|
||||
CEL: &resourceapi.CELDeviceSelector{
|
||||
Expression: "false",
|
||||
},
|
||||
}}
|
||||
class, err = f.ClientSet.ResourceV1beta1().DeviceClasses().Update(ctx, class, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
// Now create the pod.
|
||||
objects = append(objects, template, pod)
|
||||
b.create(ctx, objects...)
|
||||
|
||||
framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace))
|
||||
|
||||
// Unblock the pod.
|
||||
class.Spec.Selectors = originalClass.Spec.Selectors
|
||||
_, err = f.ClientSet.ResourceV1beta1().DeviceClasses().Update(ctx, class, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
|
||||
})
|
||||
|
||||
ginkgo.It("runs a pod without a generated resource claim", func(ctx context.Context) {
|
||||
pod, _ /* template */ := b.podInline()
|
||||
created := b.create(ctx, pod)
|
||||
pod = created[0].(*v1.Pod)
|
||||
|
||||
// Normally, this pod would be stuck because the
|
||||
// ResourceClaim cannot be created without the
|
||||
// template. We allow it to run by communicating
|
||||
// through the status that the ResourceClaim is not
|
||||
// needed.
|
||||
pod.Status.ResourceClaimStatuses = []v1.PodResourceClaimStatus{
|
||||
{Name: pod.Spec.ResourceClaims[0].Name, ResourceClaimName: nil},
|
||||
}
|
||||
_, err := f.ClientSet.CoreV1().Pods(pod.Namespace).UpdateStatus(ctx, pod, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod))
|
||||
})
|
||||
|
||||
claimTests(b, driver)
|
||||
}
|
||||
|
||||
// The following tests only make sense when there is more than one node.
|
||||
// They get skipped when there's only one node.
|
||||
multiNodeTests := func() {
|
||||
nodes := e2edra.NewNodes(f, 3, 8)
|
||||
|
||||
ginkgo.Context("with node-local resources", func() {
|
||||
driver := e2edra.NewDriver(f, nodes, perNode(1, nodes))
|
||||
b := newBuilder(f, driver)
|
||||
|
||||
ginkgo.It("uses all resources", func(ctx context.Context) {
|
||||
var objs []klog.KMetadata
|
||||
var pods []*v1.Pod
|
||||
for i := 0; i < len(nodes.NodeNames); i++ {
|
||||
pod, template := b.podInline()
|
||||
pods = append(pods, pod)
|
||||
objs = append(objs, pod, template)
|
||||
}
|
||||
b.create(ctx, objs...)
|
||||
|
||||
for _, pod := range pods {
|
||||
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
|
||||
framework.ExpectNoError(err, "start pod")
|
||||
}
|
||||
|
||||
// The pods all should run on different
|
||||
// nodes because the maximum number of
|
||||
// claims per node was limited to 1 for
|
||||
// this test.
|
||||
//
|
||||
// We cannot know for sure why the pods
|
||||
// ran on two different nodes (could
|
||||
// also be a coincidence) but if they
|
||||
// don't cover all nodes, then we have
|
||||
// a problem.
|
||||
used := make(map[string]*v1.Pod)
|
||||
for _, pod := range pods {
|
||||
pod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "get pod")
|
||||
nodeName := pod.Spec.NodeName
|
||||
if other, ok := used[nodeName]; ok {
|
||||
framework.Failf("Pod %s got started on the same node %s as pod %s although claim allocation should have been limited to one claim per node.", pod.Name, nodeName, other.Name)
|
||||
}
|
||||
used[nodeName] = pod
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
ginkgo.Context("on single node", func() {
|
||||
singleNodeTests()
|
||||
})
|
||||
|
||||
ginkgo.Context("on multiple nodes", func() {
|
||||
multiNodeTests()
|
||||
})
|
||||
})
|
||||
|
||||
// perNode returns a function which can be passed to NewDriver. The nodes
|
||||
// parameter has be instantiated, but not initialized yet, so the returned
|
||||
// function has to capture it and use it when being called.
|
||||
func perNode(maxAllocations int, nodes *e2edra.Nodes) func() e2edra.Resources {
|
||||
return func() e2edra.Resources {
|
||||
return e2edra.Resources{
|
||||
NodeLocal: true,
|
||||
MaxAllocations: maxAllocations,
|
||||
Nodes: nodes.NodeNames,
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue