Support K8s 1.22 in notebook controller (kubeflow/kubeflow#6374)
Fix https://github.com/kubeflow/kubeflow/issues/6366 Migrating to Kubebuilder v3 leads to the following changes: - Add .dockerignore file. - Upgrade Go version from v1.15 to v1.17. - Adapt Makefile. - Add image (build + push) target to makefile. - Upgrade EnvTest to use K8s v1.22. - Update PROJECT template. - Migrate CRD apiVersion from v1beta to v1. - Add livenessProbe and readinessProbe to controller manager. - Upgrade controller-runtime from v0.2.0 to v0.11.0. Other changes: - Build image using public.ecr.aws registry instead of gcr.io. - Update README.md documentation. - Update 3rd party licences. - Fix notebook.spec description. - Add 3 sample notebooks (v1, v1alpha1 and v1beta1). Signed-off-by: Samuel Veloso <svelosol@redhat.com>
This commit is contained in:
Samu
GitHub
parent
22bdc7ad77
commit
0215857aa9
|
|
@ -0,0 +1,5 @@
|
|||
# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
|
||||
# Ignore build and test binaries.
|
||||
.git
|
||||
bin/
|
||||
testbin/
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
#
|
||||
# This is necessary because the Jupyter controller now depends on
|
||||
# components/common
|
||||
ARG GOLANG_VERSION=1.15
|
||||
ARG GOLANG_VERSION=1.17
|
||||
FROM golang:${GOLANG_VERSION} as builder
|
||||
|
||||
WORKDIR /workspace
|
||||
|
|
@ -23,9 +23,9 @@ WORKDIR /workspace/notebook-controller
|
|||
|
||||
# Build
|
||||
RUN if [ "$(uname -m)" = "aarch64" ]; then \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -a -o manager main.go; \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -a -mod=mod -o manager main.go; \
|
||||
else \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go; \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -mod=mod -o manager main.go; \
|
||||
fi
|
||||
|
||||
# Use distroless as minimal base image to package the manager binary
|
||||
|
|
|
|||
|
|
@ -1,18 +1,10 @@
|
|||
|
||||
# Image URL to use all building/pushing image targets
|
||||
IMG ?= gcr.io/kubeflow-images-public/notebook-controller
|
||||
IMG ?= public.ecr.aws/j1r0q0g6/notebooks/notebook-controller
|
||||
TAG ?= $(shell git describe --tags --always)
|
||||
SHELL := /bin/bash
|
||||
GOLANG_VERSION ?= 1.15
|
||||
|
||||
# Whether to use cached images with GCB
|
||||
USE_IMAGE_CACHE ?= true
|
||||
|
||||
# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
|
||||
CRD_OPTIONS ?= "crd"
|
||||
|
||||
# Based on recommendation https://sdk.operatorframework.io/docs/building-operators/golang/references/envtest-setup/
|
||||
ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
|
||||
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
|
||||
ENVTEST_K8S_VERSION = 1.22
|
||||
|
||||
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
|
||||
ifeq (,$(shell go env GOBIN))
|
||||
|
|
@ -21,77 +13,128 @@ else
|
|||
GOBIN=$(shell go env GOBIN)
|
||||
endif
|
||||
|
||||
# Setting SHELL to bash allows bash commands to be executed by recipes.
|
||||
# This is a requirement for 'setup-envtest.sh' in the test target.
|
||||
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
|
||||
SHELL = /usr/bin/env bash -o pipefail
|
||||
.SHELLFLAGS = -ec
|
||||
|
||||
.PHONY: all
|
||||
all: manager
|
||||
|
||||
# check license
|
||||
check-license:
|
||||
##@ General
|
||||
|
||||
# The help target prints out all targets with their descriptions organized
|
||||
# beneath their categories. The categories are represented by '##@' and the
|
||||
# target descriptions by '##'. The awk commands is responsible for reading the
|
||||
# entire set of makefiles included in this invocation, looking for lines of the
|
||||
# file as xyz: ## something, and then pretty-format the target and help. Then,
|
||||
# if there's a line with ##@ something, that gets pretty-printed as a category.
|
||||
# More info on the usage of ANSI control characters for terminal formatting:
|
||||
# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
|
||||
# More info on the awk command:
|
||||
# http://linuxcommand.org/lc3_adv_awk.php
|
||||
|
||||
.PHONY: help
|
||||
help: ## Display this help.
|
||||
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
|
||||
|
||||
##@ Development
|
||||
|
||||
.PHONY: check-license
|
||||
check-license: ## Check third-party license
|
||||
./third_party/check-license.sh
|
||||
|
||||
# Run tests
|
||||
test: generate fmt vet manifests
|
||||
mkdir -p ${ENVTEST_ASSETS_DIR}
|
||||
test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/a9bd9117a77a2f84bbc546e28991136fe0000dc0/hack/setup-envtest.sh
|
||||
source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR);
|
||||
go test ./api/... ./controllers/... -coverprofile cover.out
|
||||
.PHONY: manifests
|
||||
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
|
||||
$(CONTROLLER_GEN) rbac:roleName=role crd:maxDescLen=0 webhook paths="./..." output:crd:artifacts:config=config/crd/bases
|
||||
|
||||
# Build manager binary
|
||||
manager: generate fmt vet
|
||||
go build -o bin/manager main.go
|
||||
.PHONY: generate
|
||||
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
|
||||
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
|
||||
|
||||
# Run against the configured Kubernetes cluster in ~/.kube/config
|
||||
run: generate fmt vet
|
||||
go run ./main.go
|
||||
|
||||
# Install CRDs into a cluster
|
||||
install: manifests
|
||||
kubectl apply -f config/crd/bases
|
||||
|
||||
# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
|
||||
deploy: manifests
|
||||
kubectl apply -f config/crd/bases
|
||||
kustomize build config/default | kubectl apply -f -
|
||||
|
||||
# Generate manifests e.g. CRD, RBAC etc.
|
||||
manifests: controller-gen
|
||||
$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
|
||||
|
||||
# Run go fmt against code
|
||||
fmt:
|
||||
.PHONY: fmt
|
||||
fmt: ## Run go fmt against code.
|
||||
go fmt ./...
|
||||
|
||||
# Run go vet against code
|
||||
vet:
|
||||
.PHONY: vet
|
||||
vet: ## Run go vet against code.
|
||||
go vet ./...
|
||||
|
||||
# Generate code
|
||||
generate: controller-gen
|
||||
$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths=./api/...
|
||||
.PHONY: test
|
||||
test: manifests generate fmt vet envtest ## Run tests.
|
||||
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
|
||||
|
||||
# Build the docker image
|
||||
docker-build: test
|
||||
##@ Build
|
||||
|
||||
.PHONY: manager
|
||||
manager: generate fmt vet ## Build manager binary.
|
||||
go build -o bin/manager main.go
|
||||
|
||||
.PHONY: run
|
||||
run: manifests generate fmt vet ## Run a controller from your host.
|
||||
go run ./main.go
|
||||
|
||||
.PHONY: docker-build
|
||||
docker-build: test ## Build docker image with the manager.
|
||||
cd .. && docker build . -t ${IMG}:${TAG} -f ./notebook-controller/Dockerfile
|
||||
@echo "updating kustomize image patch file for manager resource"
|
||||
sed -i'' -e 's@image: .*@image: '"${IMG}:${TAG}"'@' ./config/default/manager_image_patch.yaml
|
||||
|
||||
# Push the docker image
|
||||
docker-push:
|
||||
.PHONY: docker-push
|
||||
docker-push: ## Push docker image with the manager.
|
||||
docker push ${IMG}:${TAG}
|
||||
|
||||
# find or download controller-gen
|
||||
# download controller-gen if necessary
|
||||
controller-gen:
|
||||
ifeq (, $(shell which controller-gen))
|
||||
go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.0
|
||||
CONTROLLER_GEN=$(GOBIN)/controller-gen
|
||||
else
|
||||
CONTROLLER_GEN=$(shell which controller-gen)
|
||||
.PHONY: image
|
||||
image: docker-build docker-push ## Build and push docker image with the manager.
|
||||
|
||||
##@ Deployment
|
||||
|
||||
ifndef ignore-not-found
|
||||
ignore-not-found = false
|
||||
endif
|
||||
|
||||
# TODO(jlewi): Can we get rid of this and just use skaffold?
|
||||
build-gcr: test
|
||||
docker build -t $(IMG):$(TAG) .
|
||||
@echo Built $(IMG):$(TAG)
|
||||
.PHONY: install
|
||||
install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
|
||||
$(KUSTOMIZE) build config/crd | kubectl apply -f -
|
||||
|
||||
push-gcr: build-gcr
|
||||
docker push $(IMG):$(TAG)
|
||||
@echo Pushed $(IMG):$(TAG)
|
||||
.PHONY: uninstall
|
||||
uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
|
||||
$(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
|
||||
|
||||
.PHONY: deploy
|
||||
deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
|
||||
sed -i'' -e 's@newName: .*@newName: '"${IMG}"'@' ./config/base/kustomization.yaml
|
||||
sed -i'' -e 's@newTag: .*@newTag: '"${TAG}"'@' ./config/base/kustomization.yaml
|
||||
$(KUSTOMIZE) build config/base | kubectl apply -f -
|
||||
|
||||
.PHONY: undeploy
|
||||
undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
|
||||
$(KUSTOMIZE) build config/base | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
|
||||
|
||||
CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
|
||||
.PHONY: controller-gen
|
||||
controller-gen: ## Download controller-gen locally if necessary.
|
||||
$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0)
|
||||
|
||||
KUSTOMIZE = $(shell pwd)/bin/kustomize
|
||||
.PHONY: kustomize
|
||||
kustomize: ## Download kustomize locally if necessary.
|
||||
$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/v3/cmd/kustomize@v3.2.0)
|
||||
|
||||
ENVTEST = $(shell pwd)/bin/setup-envtest
|
||||
.PHONY: envtest
|
||||
envtest: ## Download envtest-setup locally if necessary.
|
||||
$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
|
||||
|
||||
# go-get-tool will 'go get' any package $2 and install it to $1.
|
||||
PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
|
||||
define go-get-tool
|
||||
@[ -f $(1) ] || { \
|
||||
set -e ;\
|
||||
TMP_DIR=$$(mktemp -d) ;\
|
||||
cd $$TMP_DIR ;\
|
||||
go mod init tmp ;\
|
||||
echo "Downloading $(2)" ;\
|
||||
GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
|
||||
rm -rf $$TMP_DIR ;\
|
||||
}
|
||||
endef
|
||||
|
|
|
|||
|
|
@ -1,12 +1,29 @@
|
|||
version: "2"
|
||||
domain: kubeflow.org
|
||||
layout:
|
||||
- go.kubebuilder.io/v3
|
||||
projectName: notebook-controller
|
||||
repo: github.com/kubeflow/kubeflow/components/notebook-controller
|
||||
resources:
|
||||
- group: kubeflow.org
|
||||
version: v1alpha1
|
||||
- api:
|
||||
crdVersion: v1
|
||||
namespaced: true
|
||||
controller: true
|
||||
domain: kubeflow.org
|
||||
kind: Notebook
|
||||
- group: kubeflow.org
|
||||
version: v1beta1
|
||||
kind: Notebook
|
||||
- group: kubeflow.org
|
||||
path: github.com/kubeflow/kubeflow/components/notebook-controller/api/v1
|
||||
version: v1
|
||||
- api:
|
||||
crdVersion: v1
|
||||
namespaced: true
|
||||
domain: kubeflow.org
|
||||
kind: Notebook
|
||||
path: github.com/kubeflow/kubeflow/components/notebook-controller/api/v1alpha1
|
||||
version: v1alpha1
|
||||
- api:
|
||||
crdVersion: v1
|
||||
namespaced: true
|
||||
domain: kubeflow.org
|
||||
kind: Notebook
|
||||
path: github.com/kubeflow/kubeflow/components/notebook-controller/api/v1beta1
|
||||
version: v1beta1
|
||||
version: "3"
|
||||
|
|
|
|||
|
|
@ -1,31 +1,39 @@
|
|||
# Notebook Controller
|
||||
|
||||
The controller allows users to create a custom resource "Notebook" (jupyter notebook).
|
||||
We originally wrote the controller using jsonnet and metacontroller, but are migrating to golang and
|
||||
Kubebuilder here. See [discussion](https://github.com/kubeflow/kubeflow/issues/2269).
|
||||
The controller allows users to create a custom resource "Notebook" (jupyter
|
||||
notebook).
|
||||
|
||||
It has been developed using **Golang** and
|
||||
**[Kubebuilder](https://book.kubebuilder.io/quick-start.html)**.
|
||||
|
||||
## Spec
|
||||
|
||||
The user needs to specify the PodSpec for the jupyter notebook.
|
||||
The user needs to specify the PodSpec for the Jupyter notebook.
|
||||
For example:
|
||||
|
||||
```
|
||||
apiVersion: kubeflow.org/v1alpha1
|
||||
```yaml
|
||||
apiVersion: kubeflow.org/v1
|
||||
kind: Notebook
|
||||
metadata:
|
||||
name: my-notebook
|
||||
namespace: test
|
||||
spec:
|
||||
template:
|
||||
spec: # Your PodSpec here
|
||||
spec:
|
||||
containers:
|
||||
- image: gcr.io/kubeflow-images-public/tensorflow-1.10.1-notebook-cpu:v0.3.0
|
||||
args: ["start.sh", "lab", "--LabApp.token=''", "--LabApp.allow_remote_access='True'",
|
||||
"--LabApp.allow_root='True'", "--LabApp.ip='*'",
|
||||
"--LabApp.base_url=/test/my-notebook/",
|
||||
"--port=8888", "--no-browser"]
|
||||
name: notebook
|
||||
...
|
||||
- name: my-notebook
|
||||
image: public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master
|
||||
args:
|
||||
[
|
||||
"start.sh",
|
||||
"lab",
|
||||
"--LabApp.token=''",
|
||||
"--LabApp.allow_remote_access='True'",
|
||||
"--LabApp.allow_root='True'",
|
||||
"--LabApp.ip='*'",
|
||||
"--LabApp.base_url=/test/my-notebook/",
|
||||
"--port=8888",
|
||||
"--no-browser",
|
||||
]
|
||||
```
|
||||
|
||||
The required fields are `containers[0].image` and (`containers[0].command` and/or `containers[0].args`).
|
||||
|
|
@ -45,6 +53,8 @@ All other fields will be filled in with default value if not specified.
|
|||
|
||||
`metrics-addr`: The address the metric endpoint binds to. The default value is `:8080`.
|
||||
|
||||
`probe-addr`: The address the health endpoint binds to. The default value is `:8081`.
|
||||
|
||||
`enable-leader-election`: Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. The default value is `false`.
|
||||
|
||||
## Implementation detail
|
||||
|
|
@ -53,6 +63,42 @@ This part is WIP as we are still developing.
|
|||
|
||||
Under the hood, the controller creates a StatefulSet to run the notebook instance, and a Service for it.
|
||||
|
||||
## Deployment
|
||||
|
||||
Install the `notebooks.kubeflow.org` CRD:
|
||||
|
||||
```
|
||||
make install
|
||||
```
|
||||
|
||||
Deploy the notebook controller manager:
|
||||
|
||||
```
|
||||
make deploy
|
||||
```
|
||||
|
||||
Verify that the controller is running in the `notebook-controller-system` namespace:
|
||||
|
||||
```
|
||||
$ kubectl get pods -l app=notebook-controller -n notebook-controller-system
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
notebook-controller-deployment-564d76877-mqsm8 1/1 Running 0 16s
|
||||
```
|
||||
|
||||
### Clean-up
|
||||
|
||||
Uninstall the notebook controller manager:
|
||||
|
||||
```
|
||||
make undeploy
|
||||
```
|
||||
|
||||
Uninstall the `notebooks.kubeflow.org` CRD:
|
||||
|
||||
```
|
||||
make uninstall
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
[https://www.kubeflow.org/docs/about/contributing/](https://www.kubeflow.org/docs/about/contributing/)
|
||||
|
|
@ -61,12 +107,12 @@ Under the hood, the controller creates a StatefulSet to run the notebook instanc
|
|||
|
||||
To develop on `notebook-controller`, your environment must have the following:
|
||||
|
||||
- [go](https://golang.org/dl/) version v1.15+.
|
||||
- [docker](https://docs.docker.com/install/) version 17.03+.
|
||||
- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version v1.11.3+.
|
||||
- [kustomize](https://sigs.k8s.io/kustomize/docs/INSTALL.md) v3.1.0+
|
||||
- Access to a Kubernetes v1.11.3+ cluster.
|
||||
- [kubebuilder](https://book.kubebuilder.io/quick-start.html#installation)
|
||||
- [go](https://golang.org/dl/) version v1.17+.
|
||||
- [docker](https://docs.docker.com/install/) version 20.10+.
|
||||
- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version 1.22.0+.
|
||||
- [kustomize](https://sigs.k8s.io/kustomize/docs/INSTALL.md) version 3.8.7+.
|
||||
- [kubernetes](https://github.com/kubernetes-sigs/kind) Access to a Kubernetes v1.22.0+ cluster.
|
||||
- [kubebuilder](https://book.kubebuilder.io/quick-start.html#installation) version 3.3.0+.
|
||||
|
||||
In order for the custom Notebook Controller to be functional from your local machine,
|
||||
the admins must:
|
||||
|
|
@ -79,6 +125,20 @@ the admins must:
|
|||
```
|
||||
kubectl proxy
|
||||
```
|
||||
3. Start the manager locally in developer mode:
|
||||
```
|
||||
export DEV="true"
|
||||
make run
|
||||
```
|
||||
|
||||
### Testing
|
||||
|
||||
Make sure all the tests are passing after you add a new feature:
|
||||
|
||||
```
|
||||
make test
|
||||
```
|
||||
|
||||
## TODO
|
||||
|
||||
- e2e test (we have one testing the jsonnet-metacontroller one, we should make it run on this one)
|
||||
|
|
|
|||
|
|
@ -25,8 +25,7 @@ import (
|
|||
|
||||
// NotebookSpec defines the desired state of Notebook
|
||||
type NotebookSpec struct {
|
||||
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
|
||||
// Important: Run "make" to regenerate code after modifying this file
|
||||
// Template describes the notebooks that will be created.
|
||||
Template NotebookTemplateSpec `json:"template,omitempty"`
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
//go:build !ignore_autogenerated
|
||||
// +build !ignore_autogenerated
|
||||
|
||||
/*
|
||||
|
|
@ -70,7 +71,7 @@ func (in *NotebookCondition) DeepCopy() *NotebookCondition {
|
|||
func (in *NotebookList) DeepCopyInto(out *NotebookList) {
|
||||
*out = *in
|
||||
out.TypeMeta = in.TypeMeta
|
||||
out.ListMeta = in.ListMeta
|
||||
in.ListMeta.DeepCopyInto(&out.ListMeta)
|
||||
if in.Items != nil {
|
||||
in, out := &in.Items, &out.Items
|
||||
*out = make([]Notebook, len(*in))
|
||||
|
|
|
|||
|
|
@ -25,8 +25,7 @@ import (
|
|||
|
||||
// NotebookSpec defines the desired state of Notebook
|
||||
type NotebookSpec struct {
|
||||
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
|
||||
// Important: Run "make" to regenerate code after modifying this file
|
||||
// Template describes the notebooks that will be created.
|
||||
Template NotebookTemplateSpec `json:"template,omitempty"`
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
//go:build !ignore_autogenerated
|
||||
// +build !ignore_autogenerated
|
||||
|
||||
/*
|
||||
|
|
@ -70,7 +71,7 @@ func (in *NotebookCondition) DeepCopy() *NotebookCondition {
|
|||
func (in *NotebookList) DeepCopyInto(out *NotebookList) {
|
||||
*out = *in
|
||||
out.TypeMeta = in.TypeMeta
|
||||
out.ListMeta = in.ListMeta
|
||||
in.ListMeta.DeepCopyInto(&out.ListMeta)
|
||||
if in.Items != nil {
|
||||
in, out := &in.Items, &out.Items
|
||||
*out = make([]Notebook, len(*in))
|
||||
|
|
|
|||
|
|
@ -25,8 +25,7 @@ import (
|
|||
|
||||
// NotebookSpec defines the desired state of Notebook
|
||||
type NotebookSpec struct {
|
||||
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
|
||||
// Important: Run "make" to regenerate code after modifying this file
|
||||
// Template describes the notebooks that will be created.
|
||||
Template NotebookTemplateSpec `json:"template,omitempty"`
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ package v1beta1
|
|||
|
||||
import (
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
// log is for logging in this package.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
//go:build !ignore_autogenerated
|
||||
// +build !ignore_autogenerated
|
||||
|
||||
/*
|
||||
|
|
@ -70,7 +71,7 @@ func (in *NotebookCondition) DeepCopy() *NotebookCondition {
|
|||
func (in *NotebookList) DeepCopyInto(out *NotebookList) {
|
||||
*out = *in
|
||||
out.TypeMeta = in.TypeMeta
|
||||
out.ListMeta = in.ListMeta
|
||||
in.ListMeta.DeepCopyInto(&out.ListMeta)
|
||||
if in.Items != nil {
|
||||
in, out := &in.Items, &out.Items
|
||||
*out = make([]Notebook, len(*in))
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -8,16 +8,6 @@ resources:
|
|||
patchesStrategicMerge:
|
||||
- patches/trivial_conversion_patch.yaml
|
||||
|
||||
patchesJson6902:
|
||||
# Remove once the following issue is resolved:
|
||||
# https://github.com/kubeflow/kubeflow/issues/5722
|
||||
- path: patches/old_crd.yaml
|
||||
target:
|
||||
group: apiextensions.k8s.io
|
||||
version: v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
name: notebooks.kubeflow.org
|
||||
|
||||
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
|
||||
# patches here are for enabling the conversion webhook for each CRD
|
||||
#- patches/webhook_in_notebooks.yaml
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# The following patch adds a directive for certmanager to inject CA into the CRD
|
||||
# CRD conversion requires k8s 1.13 or later.
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
annotations:
|
||||
|
|
|
|||
|
|
@ -1,64 +0,0 @@
|
|||
# Use the old CRD because of the quantity validation issue:
|
||||
# https://github.com/kubeflow/kubeflow/issues/5722
|
||||
- op: replace
|
||||
path: /spec
|
||||
value:
|
||||
group: kubeflow.org
|
||||
names:
|
||||
kind: Notebook
|
||||
plural: notebooks
|
||||
singular: notebook
|
||||
scope: Namespaced
|
||||
subresources:
|
||||
status: {}
|
||||
versions:
|
||||
- name: v1alpha1
|
||||
served: true
|
||||
storage: false
|
||||
- name: v1beta1
|
||||
served: true
|
||||
storage: false
|
||||
- name: v1
|
||||
served: true
|
||||
storage: true
|
||||
validation:
|
||||
openAPIV3Schema:
|
||||
properties:
|
||||
apiVersion:
|
||||
description: 'APIVersion defines the versioned schema of this representation
|
||||
of an object. Servers should convert recognized schemas to the latest
|
||||
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
|
||||
type: string
|
||||
kind:
|
||||
description: 'Kind is a string value representing the REST resource this
|
||||
object represents. Servers may infer this from the endpoint the client
|
||||
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
|
||||
type: string
|
||||
metadata:
|
||||
type: object
|
||||
spec:
|
||||
properties:
|
||||
template:
|
||||
description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
|
||||
Important: Run "make" to regenerate code after modifying this file'
|
||||
properties:
|
||||
spec:
|
||||
type: object
|
||||
type: object
|
||||
type: object
|
||||
status:
|
||||
properties:
|
||||
conditions:
|
||||
description: Conditions is an array of current conditions
|
||||
items:
|
||||
properties:
|
||||
type:
|
||||
description: Type of the confition/
|
||||
type: string
|
||||
required:
|
||||
- type
|
||||
type: object
|
||||
type: array
|
||||
required:
|
||||
- conditions
|
||||
type: object
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: notebooks.kubeflow.org
|
||||
spec:
|
||||
preserveUnknownFields: false # TODO: Remove in Kubeflow 1.7 release
|
||||
conversion:
|
||||
strategy: None
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# The following patch enables conversion webhook for CRD
|
||||
# CRD conversion requires k8s 1.13 or later.
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: notebooks.kubeflow.org
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ commonLabels:
|
|||
|
||||
|
||||
bases:
|
||||
- ../crd
|
||||
- ../rbac
|
||||
- ../manager
|
||||
- ../crd
|
||||
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml
|
||||
#- ../webhook
|
||||
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
|
||||
|
|
|
|||
|
|
@ -35,8 +35,14 @@ spec:
|
|||
imagePullPolicy: Always
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
path: /healthz
|
||||
port: 8081
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: 8081
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
serviceAccountName: service-account
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
apiVersion: kubeflow.org/v1
|
||||
kind: Notebook
|
||||
metadata:
|
||||
name: notebook-sample-v1
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: notebook-sample-v1
|
||||
image: public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:v1.5.0-rc.1
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
apiVersion: kubeflow.org/v1alpha1
|
||||
kind: Notebook
|
||||
metadata:
|
||||
name: notebook-sample-v1alpha1
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: notebook-sample-v1
|
||||
image: public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:v1.5.0-rc.1
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
apiVersion: kubeflow.org/v1beta1
|
||||
kind: Notebook
|
||||
metadata:
|
||||
name: notebook-sample-v1beta1
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: notebook-sample-v1
|
||||
image: public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:v1.5.0-rc.1
|
||||
|
|
@ -37,6 +37,7 @@ import (
|
|||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/tools/record"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/builder"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/event"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
|
|
@ -84,8 +85,7 @@ type NotebookReconciler struct {
|
|||
// +kubebuilder:rbac:groups=kubeflow.org,resources=notebooks;notebooks/status;notebooks/finalizers,verbs="*"
|
||||
// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs="*"
|
||||
|
||||
func (r *NotebookReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
|
||||
ctx := context.Background()
|
||||
func (r *NotebookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
log := r.Log.WithValues("notebook", req.NamespacedName)
|
||||
|
||||
// TODO(yanniszark): Can we avoid reconciling Events and Notebook in the same queue?
|
||||
|
|
@ -628,11 +628,79 @@ func nbNameExists(client client.Client, nbName string, namespace string) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
// predNBPodIsLabeled filters pods not containing the "notebook-name" label key
|
||||
func predNBPodIsLabeled() predicate.Funcs {
|
||||
// Documented at
|
||||
// https://github.com/kubernetes-sigs/controller-runtime/blob/ce8bdd3d81ab410ff23255e9ad3554f613c5183c/pkg/predicate/predicate_test.go#L884
|
||||
checkNBLabel := func() func(object client.Object) bool {
|
||||
return func(object client.Object) bool {
|
||||
_, labelExists := object.GetLabels()["notebook-name"]
|
||||
return labelExists
|
||||
}
|
||||
}
|
||||
|
||||
return predicate.NewPredicateFuncs(checkNBLabel())
|
||||
}
|
||||
|
||||
// predNBEvents filters events not coming from Pod or STS, and coming from
|
||||
// unknown NBs
|
||||
func predNBEvents(r *NotebookReconciler) predicate.Funcs {
|
||||
checkEvent := func() func(object client.Object) bool {
|
||||
return func(object client.Object) bool {
|
||||
event := object.(*corev1.Event)
|
||||
nbName, err := nbNameFromInvolvedObject(r.Client, &event.InvolvedObject)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return isStsOrPodEvent(event) && nbNameExists(r.Client, nbName, object.GetNamespace())
|
||||
}
|
||||
}
|
||||
|
||||
predicates := predicate.NewPredicateFuncs(checkEvent())
|
||||
|
||||
// Do not reconcile when an event gets deleted
|
||||
predicates.DeleteFunc = func(e event.DeleteEvent) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
return predicates
|
||||
}
|
||||
|
||||
// SetupWithManager sets up the controller with the Manager.
|
||||
func (r *NotebookReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
|
||||
// Map function to convert pod events to reconciliation requests
|
||||
mapPodToRequest := func(object client.Object) []reconcile.Request {
|
||||
return []reconcile.Request{
|
||||
{NamespacedName: types.NamespacedName{
|
||||
Name: object.GetLabels()["notebook-name"],
|
||||
Namespace: object.GetNamespace(),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// Map function to convert namespace events to reconciliation requests
|
||||
mapEventToRequest := func(object client.Object) []reconcile.Request {
|
||||
return []reconcile.Request{
|
||||
{NamespacedName: types.NamespacedName{
|
||||
Name: object.GetName(),
|
||||
Namespace: object.GetNamespace(),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
builder := ctrl.NewControllerManagedBy(mgr).
|
||||
For(&v1beta1.Notebook{}).
|
||||
Owns(&appsv1.StatefulSet{}).
|
||||
Owns(&corev1.Service{})
|
||||
Owns(&corev1.Service{}).
|
||||
Watches(
|
||||
&source.Kind{Type: &corev1.Pod{}},
|
||||
handler.EnqueueRequestsFromMapFunc(mapPodToRequest),
|
||||
builder.WithPredicates(predNBPodIsLabeled())).
|
||||
Watches(
|
||||
&source.Kind{Type: &corev1.Event{}},
|
||||
handler.EnqueueRequestsFromMapFunc(mapEventToRequest),
|
||||
builder.WithPredicates(predNBEvents(r)))
|
||||
// watch Istio virtual service
|
||||
if os.Getenv("USE_ISTIO") == "true" {
|
||||
virtualService := &unstructured.Unstructured{}
|
||||
|
|
@ -641,100 +709,10 @@ func (r *NotebookReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
|||
builder.Owns(virtualService)
|
||||
}
|
||||
|
||||
// TODO(lunkai): After this is fixed:
|
||||
// https://github.com/kubernetes-sigs/controller-runtime/issues/572
|
||||
// We don't have to call Build to get the controller.
|
||||
c, err := builder.Build(r)
|
||||
err := builder.Complete(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// watch underlying pod
|
||||
mapFn := handler.ToRequestsFunc(
|
||||
func(a handler.MapObject) []ctrl.Request {
|
||||
return []ctrl.Request{
|
||||
{NamespacedName: types.NamespacedName{
|
||||
Name: a.Meta.GetLabels()["notebook-name"],
|
||||
Namespace: a.Meta.GetNamespace(),
|
||||
}},
|
||||
}
|
||||
})
|
||||
|
||||
// helper common function for pod predicates. Filter pods not containing the "notebook-name" label key
|
||||
checkNBLabel := func(m metav1.Object) bool {
|
||||
_, ok := m.GetLabels()["notebook-name"]
|
||||
return ok
|
||||
}
|
||||
// TODO: refactor to use predicate.NewPredicateFuncs when controller-runtime module version is updated
|
||||
p := predicate.Funcs{
|
||||
UpdateFunc: func(e event.UpdateEvent) bool {
|
||||
return checkNBLabel(e.MetaOld) && e.ObjectOld != e.ObjectNew
|
||||
},
|
||||
CreateFunc: func(e event.CreateEvent) bool {
|
||||
return checkNBLabel(e.Meta)
|
||||
},
|
||||
DeleteFunc: func(e event.DeleteEvent) bool {
|
||||
return checkNBLabel(e.Meta)
|
||||
},
|
||||
GenericFunc: func(e event.GenericEvent) bool {
|
||||
return checkNBLabel(e.Meta)
|
||||
},
|
||||
}
|
||||
|
||||
eventToRequest := handler.ToRequestsFunc(
|
||||
func(a handler.MapObject) []ctrl.Request {
|
||||
return []reconcile.Request{
|
||||
{NamespacedName: types.NamespacedName{
|
||||
Name: a.Meta.GetName(),
|
||||
Namespace: a.Meta.GetNamespace(),
|
||||
}},
|
||||
}
|
||||
})
|
||||
|
||||
// helper common function for event predicates. Filter events not coming from Pod or STS, and coming from unknown NBs
|
||||
checkEvent := func(o runtime.Object, m metav1.Object) bool {
|
||||
event := o.(*corev1.Event)
|
||||
nbName, err := nbNameFromInvolvedObject(r.Client, &event.InvolvedObject)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return isStsOrPodEvent(event) && nbNameExists(r.Client, nbName, m.GetNamespace())
|
||||
}
|
||||
// TODO: refactor to use predicate.NewPredicateFuncs when controller-runtime module version is updated
|
||||
// We don't include DeleteFunc since we don't want the reconcile
|
||||
// to happen when an event gets deleted
|
||||
eventsPredicates := predicate.Funcs{
|
||||
UpdateFunc: func(e event.UpdateEvent) bool {
|
||||
return e.ObjectOld != e.ObjectNew && checkEvent(e.ObjectNew, e.MetaNew)
|
||||
},
|
||||
CreateFunc: func(e event.CreateEvent) bool {
|
||||
return checkEvent(e.Object, e.Meta)
|
||||
},
|
||||
DeleteFunc: func(e event.DeleteEvent) bool {
|
||||
return false
|
||||
},
|
||||
GenericFunc: func(e event.GenericEvent) bool {
|
||||
return checkEvent(e.Object, e.Meta)
|
||||
},
|
||||
}
|
||||
|
||||
if err = c.Watch(
|
||||
&source.Kind{Type: &corev1.Pod{}},
|
||||
&handler.EnqueueRequestsFromMapFunc{
|
||||
ToRequests: mapFn,
|
||||
},
|
||||
p); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = c.Watch(
|
||||
&source.Kind{Type: &corev1.Event{}},
|
||||
&handler.EnqueueRequestsFromMapFunc{
|
||||
ToRequests: eventToRequest,
|
||||
},
|
||||
eventsPredicates); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ limitations under the License.
|
|||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
|
|
@ -39,22 +40,28 @@ import (
|
|||
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
|
||||
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
|
||||
|
||||
var cfg *rest.Config
|
||||
var k8sClient client.Client // You'll be using this client in your tests.
|
||||
var testEnv *envtest.Environment
|
||||
var (
|
||||
cfg *rest.Config
|
||||
k8sClient client.Client
|
||||
testEnv *envtest.Environment
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
)
|
||||
|
||||
var _ = BeforeSuite(func(done Done) {
|
||||
logf.SetLogger(zap.LoggerTo(GinkgoWriter, true))
|
||||
var _ = BeforeSuite(func() {
|
||||
logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
|
||||
|
||||
ctx, cancel = context.WithCancel(context.TODO())
|
||||
|
||||
By("bootstrapping test environment")
|
||||
testEnv = &envtest.Environment{
|
||||
CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
|
||||
CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
|
||||
ErrorIfCRDPathMissing: true,
|
||||
}
|
||||
|
||||
var err error
|
||||
cfg, err = testEnv.Start()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(cfg).ToNot(BeNil())
|
||||
cfg, err := testEnv.Start()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(cfg).NotTo(BeNil())
|
||||
|
||||
err = nbv1beta1.AddToScheme(scheme.Scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
|
@ -62,13 +69,13 @@ var _ = BeforeSuite(func(done Done) {
|
|||
// +kubebuilder:scaffold:scheme
|
||||
|
||||
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(k8sClient).ToNot(BeNil())
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(k8sClient).NotTo(BeNil())
|
||||
|
||||
k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{
|
||||
Scheme: scheme.Scheme,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = (&NotebookReconciler{
|
||||
Client: k8sManager.GetClient(),
|
||||
|
|
@ -80,20 +87,21 @@ var _ = BeforeSuite(func(done Done) {
|
|||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
go func() {
|
||||
err = k8sManager.Start(ctrl.SetupSignalHandler())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
defer GinkgoRecover()
|
||||
err = k8sManager.Start(ctx)
|
||||
Expect(err).ToNot(HaveOccurred(), "failed to run manager")
|
||||
}()
|
||||
|
||||
k8sClient = k8sManager.GetClient()
|
||||
Expect(k8sClient).ToNot(BeNil())
|
||||
|
||||
close(done)
|
||||
}, 60)
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
cancel()
|
||||
By("tearing down the test environment")
|
||||
err := testEnv.Stop()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
func TestAPIs(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -1,21 +1,75 @@
|
|||
module github.com/kubeflow/kubeflow/components/notebook-controller
|
||||
|
||||
go 1.15
|
||||
go 1.17
|
||||
|
||||
require (
|
||||
github.com/go-logr/logr v0.1.0
|
||||
github.com/kubeflow/kubeflow/components/common v0.0.0-20200908101143-7f5e242f4671
|
||||
github.com/prometheus/client_golang v0.9.0
|
||||
github.com/onsi/ginkgo v1.12.1
|
||||
github.com/onsi/gomega v1.10.1
|
||||
k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b
|
||||
k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d
|
||||
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
|
||||
sigs.k8s.io/controller-runtime v0.2.0
|
||||
sigs.k8s.io/controller-tools v0.2.0 // indirect
|
||||
github.com/go-logr/logr v1.2.0
|
||||
github.com/kubeflow/kubeflow/components/common v0.0.0-20220218084159-4ad0158e955e
|
||||
github.com/onsi/ginkgo v1.16.5
|
||||
github.com/onsi/gomega v1.17.0
|
||||
github.com/prometheus/client_golang v1.11.0
|
||||
k8s.io/api v0.23.0
|
||||
k8s.io/apimachinery v0.23.0
|
||||
k8s.io/client-go v0.23.0
|
||||
sigs.k8s.io/controller-runtime v0.11.0
|
||||
)
|
||||
|
||||
// Ensure we build the notebook-controller with the latest `common`
|
||||
// module. However, because this module's `replace` will be ignored by
|
||||
// other modules, we still specify a commit in the `require` directive.
|
||||
replace github.com/kubeflow/kubeflow/components/common => ../common
|
||||
require (
|
||||
cloud.google.com/go v0.81.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.11.18 // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
|
||||
github.com/Azure/go-autorest/logger v0.2.1 // indirect
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
|
||||
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
|
||||
github.com/fsnotify/fsnotify v1.5.1 // indirect
|
||||
github.com/go-logr/zapr v1.2.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/go-cmp v0.5.5 // indirect
|
||||
github.com/google/gofuzz v1.1.0 // indirect
|
||||
github.com/google/uuid v1.1.2 // indirect
|
||||
github.com/googleapis/gnostic v0.5.5 // indirect
|
||||
github.com/imdario/mergo v0.3.12 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/nxadm/tail v1.4.8 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.28.0 // indirect
|
||||
github.com/prometheus/procfs v0.6.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
go.uber.org/zap v1.19.1 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
|
||||
golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
|
||||
golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
|
||||
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
k8s.io/apiextensions-apiserver v0.23.0 // indirect
|
||||
k8s.io/component-base v0.23.0 // indirect
|
||||
k8s.io/klog/v2 v2.30.0 // indirect
|
||||
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
|
||||
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
|
||||
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
|
||||
sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
|
||||
sigs.k8s.io/yaml v1.3.0 // indirect
|
||||
)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -19,17 +19,23 @@ import (
|
|||
"flag"
|
||||
"os"
|
||||
|
||||
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
|
||||
// to ensure that exec-entrypoint and run can make use of them.
|
||||
_ "k8s.io/client-go/plugin/pkg/client/auth"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/healthz"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
|
||||
nbv1 "github.com/kubeflow/kubeflow/components/notebook-controller/api/v1"
|
||||
nbv1alpha1 "github.com/kubeflow/kubeflow/components/notebook-controller/api/v1alpha1"
|
||||
nbv1beta1 "github.com/kubeflow/kubeflow/components/notebook-controller/api/v1beta1"
|
||||
"github.com/kubeflow/kubeflow/components/notebook-controller/controllers"
|
||||
controller_metrics "github.com/kubeflow/kubeflow/components/notebook-controller/pkg/metrics"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
// +kubebuilder:scaffold:imports
|
||||
//+kubebuilder:scaffold:imports
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -38,28 +44,36 @@ var (
|
|||
)
|
||||
|
||||
func init() {
|
||||
_ = clientgoscheme.AddToScheme(scheme)
|
||||
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
|
||||
|
||||
_ = nbv1alpha1.AddToScheme(scheme)
|
||||
_ = nbv1beta1.AddToScheme(scheme)
|
||||
_ = nbv1.AddToScheme(scheme)
|
||||
// +kubebuilder:scaffold:scheme
|
||||
utilruntime.Must(nbv1.AddToScheme(scheme))
|
||||
utilruntime.Must(nbv1alpha1.AddToScheme(scheme))
|
||||
utilruntime.Must(nbv1beta1.AddToScheme(scheme))
|
||||
|
||||
//+kubebuilder:scaffold:scheme
|
||||
}
|
||||
|
||||
func main() {
|
||||
var metricsAddr, leaderElectionNamespace string
|
||||
var enableLeaderElection bool
|
||||
var probeAddr string
|
||||
flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
|
||||
flag.StringVar(&probeAddr, "probe-addr", ":8081", "The address the health endpoint binds to.")
|
||||
flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "",
|
||||
"Determines the namespace in which the leader election configmap will be created.")
|
||||
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
|
||||
"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
|
||||
opts := zap.Options{
|
||||
Development: true,
|
||||
}
|
||||
opts.BindFlags(flag.CommandLine)
|
||||
flag.Parse()
|
||||
|
||||
ctrl.SetLogger(zap.Logger(true))
|
||||
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
|
||||
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
|
||||
Scheme: scheme,
|
||||
MetricsBindAddress: metricsAddr,
|
||||
HealthProbeBindAddress: probeAddr,
|
||||
LeaderElection: enableLeaderElection,
|
||||
LeaderElectionNamespace: leaderElectionNamespace,
|
||||
LeaderElectionID: "kubeflow-notebook-controller",
|
||||
|
|
@ -80,13 +94,23 @@ func main() {
|
|||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
|
||||
setupLog.Error(err, "unable to set up health check")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
|
||||
setupLog.Error(err, "unable to set up ready check")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// uncomment when we need the conversion webhook.
|
||||
// if err = (&nbv1beta1.Notebook{}).SetupWebhookWithManager(mgr); err != nil {
|
||||
// setupLog.Error(err, "unable to create webhook", "webhook", "Captain")
|
||||
// os.Exit(1)
|
||||
// }
|
||||
|
||||
// +kubebuilder:scaffold:builder
|
||||
//+kubebuilder:scaffold:builder
|
||||
|
||||
setupLog.Info("starting manager")
|
||||
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import (
|
|||
"github.com/kubeflow/kubeflow/components/notebook-controller/pkg/metrics"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
var log = logf.Log.WithName("culler")
|
||||
|
|
|
|||
|
|
@ -1,66 +1,291 @@
|
|||
github.com/kubeflow/kubeflow/components/notebook-controller
|
||||
cloud.google.com/go
|
||||
cloud.google.com/go/bigquery
|
||||
cloud.google.com/go/datastore
|
||||
cloud.google.com/go/firestore
|
||||
cloud.google.com/go/pubsub
|
||||
cloud.google.com/go/storage
|
||||
dmitri.shuralyov.com/gpu/mtl
|
||||
github.com/Azure/go-ansiterm
|
||||
github.com/Azure/go-autorest
|
||||
github.com/Azure/go-autorest/autorest
|
||||
github.com/Azure/go-autorest/autorest/adal
|
||||
github.com/Azure/go-autorest/autorest/date
|
||||
github.com/Azure/go-autorest/autorest/mocks
|
||||
github.com/Azure/go-autorest/logger
|
||||
github.com/Azure/go-autorest/tracing
|
||||
github.com/BurntSushi/toml
|
||||
github.com/BurntSushi/xgb
|
||||
github.com/NYTimes/gziphandler
|
||||
github.com/OneOfOne/xxhash
|
||||
github.com/PuerkitoBio/purell
|
||||
github.com/PuerkitoBio/urlesc
|
||||
github.com/alecthomas/template
|
||||
github.com/alecthomas/units
|
||||
github.com/antihax/optional
|
||||
github.com/antlr/antlr4/runtime/Go/antlr
|
||||
github.com/armon/circbuf
|
||||
github.com/armon/go-metrics
|
||||
github.com/armon/go-radix
|
||||
github.com/asaskevich/govalidator
|
||||
github.com/benbjohnson/clock
|
||||
github.com/beorn7/perks
|
||||
github.com/bgentry/speakeasy
|
||||
github.com/bketelsen/crypt
|
||||
github.com/blang/semver
|
||||
github.com/census-instrumentation/opencensus-proto
|
||||
github.com/certifi/gocertifi
|
||||
github.com/cespare/xxhash
|
||||
github.com/cespare/xxhash/v2
|
||||
github.com/chzyer/logex
|
||||
github.com/chzyer/readline
|
||||
github.com/chzyer/test
|
||||
github.com/client9/misspell
|
||||
github.com/cncf/udpa/go
|
||||
github.com/cncf/xds/go
|
||||
github.com/cockroachdb/datadriven
|
||||
github.com/cockroachdb/errors
|
||||
github.com/cockroachdb/logtags
|
||||
github.com/coreos/bbolt
|
||||
github.com/coreos/etcd
|
||||
github.com/coreos/go-oidc
|
||||
github.com/coreos/go-semver
|
||||
github.com/coreos/go-systemd
|
||||
github.com/coreos/go-systemd/v22
|
||||
github.com/coreos/pkg
|
||||
github.com/cpuguy83/go-md2man/v2
|
||||
github.com/creack/pty
|
||||
github.com/davecgh/go-spew
|
||||
github.com/dgrijalva/jwt-go
|
||||
github.com/dgryski/go-sip13
|
||||
github.com/docopt/docopt-go
|
||||
github.com/dustin/go-humanize
|
||||
github.com/elazarl/goproxy
|
||||
github.com/emicklei/go-restful
|
||||
github.com/envoyproxy/go-control-plane
|
||||
github.com/envoyproxy/protoc-gen-validate
|
||||
github.com/evanphx/json-patch
|
||||
github.com/fatih/color
|
||||
github.com/felixge/httpsnoop
|
||||
github.com/form3tech-oss/jwt-go
|
||||
github.com/fsnotify/fsnotify
|
||||
github.com/getkin/kin-openapi
|
||||
github.com/getsentry/raven-go
|
||||
github.com/ghodss/yaml
|
||||
github.com/go-gl/glfw
|
||||
github.com/go-gl/glfw/v3.3/glfw
|
||||
github.com/go-kit/kit
|
||||
github.com/go-kit/log
|
||||
github.com/go-logfmt/logfmt
|
||||
github.com/go-logr/logr
|
||||
github.com/go-logr/zapr
|
||||
github.com/go-openapi/jsonpointer
|
||||
github.com/go-openapi/jsonreference
|
||||
github.com/go-openapi/swag
|
||||
github.com/go-stack/stack
|
||||
github.com/go-task/slim-sprig
|
||||
github.com/godbus/dbus/v5
|
||||
github.com/gogo/protobuf
|
||||
github.com/golang/glog
|
||||
github.com/golang/groupcache
|
||||
github.com/golang/mock
|
||||
github.com/golang/protobuf
|
||||
github.com/google/btree
|
||||
github.com/google/cel-go
|
||||
github.com/google/cel-spec
|
||||
github.com/google/go-cmp
|
||||
github.com/google/gofuzz
|
||||
github.com/google/martian
|
||||
github.com/google/martian/v3
|
||||
github.com/google/pprof
|
||||
github.com/google/renameio
|
||||
github.com/google/uuid
|
||||
github.com/googleapis/gax-go/v2
|
||||
github.com/googleapis/gnostic
|
||||
github.com/gopherjs/gopherjs
|
||||
github.com/gorilla/mux
|
||||
github.com/gorilla/websocket
|
||||
github.com/gregjones/httpcache
|
||||
github.com/grpc-ecosystem/go-grpc-middleware
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus
|
||||
github.com/grpc-ecosystem/grpc-gateway
|
||||
github.com/hashicorp/consul/api
|
||||
github.com/hashicorp/consul/sdk
|
||||
github.com/hashicorp/errwrap
|
||||
github.com/hashicorp/go-cleanhttp
|
||||
github.com/hashicorp/go-immutable-radix
|
||||
github.com/hashicorp/go-msgpack
|
||||
github.com/hashicorp/go-multierror
|
||||
github.com/hashicorp/go-rootcerts
|
||||
github.com/hashicorp/go-sockaddr
|
||||
github.com/hashicorp/go-syslog
|
||||
github.com/hashicorp/go-uuid
|
||||
github.com/hashicorp/go.net
|
||||
github.com/hashicorp/golang-lru
|
||||
github.com/hashicorp/hcl
|
||||
github.com/hashicorp/logutils
|
||||
github.com/hashicorp/mdns
|
||||
github.com/hashicorp/memberlist
|
||||
github.com/hashicorp/serf
|
||||
github.com/hpcloud/tail
|
||||
github.com/ianlancetaylor/demangle
|
||||
github.com/imdario/mergo
|
||||
github.com/inconshreveable/mousetrap
|
||||
github.com/jessevdk/go-flags
|
||||
github.com/jonboulle/clockwork
|
||||
github.com/josharian/intern
|
||||
github.com/jpillora/backoff
|
||||
github.com/json-iterator/go
|
||||
github.com/jstemmer/go-junit-report
|
||||
github.com/jtolds/gls
|
||||
github.com/julienschmidt/httprouter
|
||||
github.com/kisielk/errcheck
|
||||
github.com/kisielk/gotool
|
||||
github.com/konsorten/go-windows-terminal-sequences
|
||||
github.com/kr/fs
|
||||
github.com/kr/logfmt
|
||||
github.com/kr/pretty
|
||||
github.com/kr/pty
|
||||
github.com/kr/text
|
||||
github.com/kubeflow/kubeflow/components/common
|
||||
github.com/magiconair/properties
|
||||
github.com/mailru/easyjson
|
||||
github.com/mattn/go-colorable
|
||||
github.com/mattn/go-isatty
|
||||
github.com/matttproud/golang_protobuf_extensions
|
||||
github.com/miekg/dns
|
||||
github.com/mitchellh/cli
|
||||
github.com/mitchellh/go-homedir
|
||||
github.com/mitchellh/go-testing-interface
|
||||
github.com/mitchellh/gox
|
||||
github.com/mitchellh/iochan
|
||||
github.com/mitchellh/mapstructure
|
||||
github.com/moby/spdystream
|
||||
github.com/moby/term
|
||||
github.com/modern-go/concurrent
|
||||
github.com/modern-go/reflect2
|
||||
github.com/munnerz/goautoneg
|
||||
github.com/mwitkow/go-conntrack
|
||||
github.com/mxk/go-flowrate
|
||||
github.com/niemeyer/pretty
|
||||
github.com/nxadm/tail
|
||||
github.com/oklog/ulid
|
||||
github.com/onsi/ginkgo
|
||||
github.com/onsi/gomega
|
||||
github.com/pborman/uuid
|
||||
github.com/opentracing/opentracing-go
|
||||
github.com/pascaldekloe/goe
|
||||
github.com/pelletier/go-toml
|
||||
github.com/peterbourgon/diskv
|
||||
github.com/pkg/errors
|
||||
github.com/pkg/sftp
|
||||
github.com/pmezard/go-difflib
|
||||
github.com/posener/complete
|
||||
github.com/pquerna/cachecontrol
|
||||
github.com/prometheus/client_golang
|
||||
github.com/prometheus/client_model
|
||||
github.com/prometheus/common
|
||||
github.com/prometheus/procfs
|
||||
github.com/prometheus/tsdb
|
||||
github.com/rogpeppe/fastuuid
|
||||
github.com/rogpeppe/go-internal
|
||||
github.com/russross/blackfriday/v2
|
||||
github.com/ryanuber/columnize
|
||||
github.com/sean-/seed
|
||||
github.com/shurcooL/sanitized_anchor_name
|
||||
github.com/sirupsen/logrus
|
||||
github.com/smartystreets/assertions
|
||||
github.com/smartystreets/goconvey
|
||||
github.com/soheilhy/cmux
|
||||
github.com/spaolacci/murmur3
|
||||
github.com/spf13/afero
|
||||
github.com/spf13/cast
|
||||
github.com/spf13/cobra
|
||||
github.com/spf13/jwalterweatherman
|
||||
github.com/spf13/pflag
|
||||
github.com/spf13/viper
|
||||
github.com/stoewer/go-strcase
|
||||
github.com/stretchr/objx
|
||||
github.com/stretchr/testify
|
||||
github.com/subosito/gotenv
|
||||
github.com/tmc/grpc-websocket-proxy
|
||||
github.com/xiang90/probing
|
||||
github.com/yuin/goldmark
|
||||
go.etcd.io/bbolt
|
||||
go.etcd.io/etcd/api/v3
|
||||
go.etcd.io/etcd/client/pkg/v3
|
||||
go.etcd.io/etcd/client/v2
|
||||
go.etcd.io/etcd/client/v3
|
||||
go.etcd.io/etcd/pkg/v3
|
||||
go.etcd.io/etcd/raft/v3
|
||||
go.etcd.io/etcd/server/v3
|
||||
go.opencensus.io
|
||||
go.opentelemetry.io/contrib
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp
|
||||
go.opentelemetry.io/otel
|
||||
go.opentelemetry.io/otel/exporters/otlp
|
||||
go.opentelemetry.io/otel/metric
|
||||
go.opentelemetry.io/otel/oteltest
|
||||
go.opentelemetry.io/otel/sdk
|
||||
go.opentelemetry.io/otel/sdk/export/metric
|
||||
go.opentelemetry.io/otel/sdk/metric
|
||||
go.opentelemetry.io/otel/trace
|
||||
go.opentelemetry.io/proto/otlp
|
||||
go.uber.org/atomic
|
||||
go.uber.org/goleak
|
||||
go.uber.org/multierr
|
||||
go.uber.org/zap
|
||||
golang.org/x/crypto
|
||||
golang.org/x/exp
|
||||
golang.org/x/image
|
||||
golang.org/x/lint
|
||||
golang.org/x/mobile
|
||||
golang.org/x/mod
|
||||
golang.org/x/net
|
||||
golang.org/x/oauth2
|
||||
golang.org/x/sync
|
||||
golang.org/x/sys
|
||||
golang.org/x/term
|
||||
golang.org/x/text
|
||||
golang.org/x/time
|
||||
golang.org/x/tools
|
||||
golang.org/x/xerrors
|
||||
gomodules.xyz/jsonpatch/v2
|
||||
google.golang.org/api
|
||||
google.golang.org/appengine
|
||||
google.golang.org/genproto
|
||||
google.golang.org/grpc
|
||||
google.golang.org/protobuf
|
||||
gopkg.in/alecthomas/kingpin.v2
|
||||
gopkg.in/check.v1
|
||||
gopkg.in/errgo.v2
|
||||
gopkg.in/fsnotify.v1
|
||||
gopkg.in/inf.v0
|
||||
gopkg.in/ini.v1
|
||||
gopkg.in/natefinch/lumberjack.v2
|
||||
gopkg.in/resty.v1
|
||||
gopkg.in/square/go-jose.v2
|
||||
gopkg.in/tomb.v1
|
||||
gopkg.in/yaml.v2
|
||||
gopkg.in/yaml.v3
|
||||
gotest.tools/v3
|
||||
honnef.co/go/tools
|
||||
k8s.io/api
|
||||
k8s.io/apiextensions-apiserver
|
||||
k8s.io/apimachinery
|
||||
k8s.io/apiserver
|
||||
k8s.io/client-go
|
||||
k8s.io/klog
|
||||
k8s.io/code-generator
|
||||
k8s.io/component-base
|
||||
k8s.io/gengo
|
||||
k8s.io/klog/v2
|
||||
k8s.io/kube-openapi
|
||||
k8s.io/utils
|
||||
rsc.io/binaryregexp
|
||||
rsc.io/quote/v3
|
||||
rsc.io/sampler
|
||||
sigs.k8s.io/apiserver-network-proxy/konnectivity-client
|
||||
sigs.k8s.io/controller-runtime
|
||||
sigs.k8s.io/testing_frameworks
|
||||
sigs.k8s.io/json
|
||||
sigs.k8s.io/structured-merge-diff/v4
|
||||
sigs.k8s.io/yaml
|
||||
|
|
|
|||
|
|
@ -464,7 +464,7 @@ modification, are permitted provided that the following conditions are met:
|
|||
|
||||
* Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright notice
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
* Neither the name of the Evan Phoenix nor the names of its contributors
|
||||
|
|
@ -3987,7 +3987,7 @@ stretchr/testify MIT License https://github.com/stretchr/testify/blob/master/L
|
|||
--------------------------------------------------------------------------------
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell
|
||||
Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
@ -4033,7 +4033,7 @@ THE SOFTWARE.
|
|||
--------------------------------------------------------------------------------
|
||||
uber-go/multierr MIT License https://github.com/uber-go/multierr/blob/master/LICENSE.txt
|
||||
--------------------------------------------------------------------------------
|
||||
Copyright (c) 2017 Uber Technologies, Inc.
|
||||
Copyright (c) 2017-2021 Uber Technologies, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
Loading…
Reference in New Issue