kubeflow
/
notebooks
mirror of https://github.com/kubeflow/notebooks.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: update notebook server images + support ARM64 (kubeflow/kubeflow#7357) 

* feat: update example notebook servers

* docs: update example notebook servers readme

* feat: update code-server notebook image start args

* docs: update links to use kubeflow/kubeflow repo
This commit is contained in:
Mathew Wicks 2023-10-24 14:34:07 -07:00 committed by GitHub
parent 0e787960f5
commit 15b1068a40
65 changed files with 1216 additions and 628 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
components/example-notebook-servers/Makefile
View File

@ -1,97 +1,49 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
# NOTE: the images must be listed in the required build order
IMAGE_FOLDERS ?= \
base \
codeserver \
codeserver-python \
jupyter \
jupyter-scipy \
jupyter-pytorch \
jupyter-pytorch-full \
jupyter-pytorch-cuda \
jupyter-pytorch-cuda-full \
jupyter-tensorflow \
jupyter-tensorflow-full \
jupyter-tensorflow-cuda \
jupyter-tensorflow-cuda-full \
rstudio \
rstudio-tidyverse \
# ------------------------------------------------------------------------------
# docker-build | build all images
#
.PHONY: docker-build
docker-build: build-jupyter-pytorch-full build-jupyter-tensorflow-full build-jupyter-scipy build-codeserver-python build-rstudio-tidyverse
@echo "\nAll notebook-server images have been successfully built...\n"
.PHONY: build-jupyter-pytorch-full
build-jupyter-pytorch-full: build-jupyter-pytorch-full-cpu build-jupyter-pytorch-full-cuda
.PHONY: build-jupyter-pytorch-full-cpu
build-jupyter-pytorch-full-cpu:
@echo "\nBuilding jupyter-pytorch-full-cpu image...\n"
$(MAKE) docker-build-cpu -C jupyter-pytorch-full TAG=${TAG}
.PHONY: build-jupyter-pytorch-full-cuda
build-jupyter-pytorch-full-cuda:
@echo "\nBuilding jupyter-pytorch-full-cuda image...\n"
$(MAKE) docker-build-cuda -C jupyter-pytorch-full TAG=${TAG}
.PHONY: build-jupyter-tensorflow-full
build-jupyter-tensorflow-full: build-jupyter-tensorflow-full-cpu build-jupyter-tensorflow-full-cuda
.PHONY: jupyter-tensorflow-full-cpu
build-jupyter-tensorflow-full-cpu:
@echo "\nBuilding jupyter-tensorflow-full-cpu image...\n"
$(MAKE) docker-build-cpu -C jupyter-tensorflow-full TAG=${TAG}
.PHONY: build-jupyter-tensorflow-full-cuda
build-jupyter-tensorflow-full-cuda:
@echo "\nBuilding jupyter-tensorflow-full-cuda image...\n"
$(MAKE) docker-build-cuda -C jupyter-tensorflow-full TAG=${TAG}
.PHONY: build-jupyter-scipy
build-jupyter-scipy:
@echo "\nBuilding jupyter-scipy image...\n"
$(MAKE) docker-build -C jupyter-scipy TAG=${TAG}
.PHONY: build-codeserver-python
build-codeserver-python:
@echo "\nBuilding codeserver-python image...\n"
$(MAKE) docker-build -C codeserver-python TAG=${TAG}
.PHONY: build-rstudio-tidyverse
build-rstudio-tidyverse:
@echo "\nBuilding rstudio-tidyverse image...\n"
$(MAKE) docker-build -C rstudio-tidyverse TAG=${TAG}
docker-build: $(addprefix docker-build--, $(IMAGE_FOLDERS))
docker-build--%:
$(MAKE) docker-build -C $*
# ------------------------------------------------------------------------------
# docker-push | push all images
#
.PHONY: docker-push
docker-push:
@echo "\nPushing base image...\n"
$(MAKE) docker-push -C base
docker-push: $(addprefix docker-push--, $(IMAGE_FOLDERS))
docker-push--%:
$(MAKE) docker-push -C $*
@echo "\nPushing codeserver image...\n"
$(MAKE) docker-push -C codeserver
# ------------------------------------------------------------------------------
# docker-build-multi-arch | multi-arch build all images
#
.PHONY: docker-build-multi-arch
docker-build-multi-arch: $(addprefix docker-build-multi-arch--, $(IMAGE_FOLDERS))
docker-build-multi-arch--%:
$(MAKE) docker-build-multi-arch -C $*
@echo "\nPushing codeserver-python image...\n"
$(MAKE) docker-push -C codeserver-python
@echo "\nPushing rstudio image...\n"
$(MAKE) docker-push -C rstudio
@echo "\nPushing rstudio-tidyverse image...\n"
$(MAKE) docker-push -C rstudio-tidyverse
@echo "\nPushing jupyter image...\n"
$(MAKE) docker-push -C jupyter
@echo "\nPushing jupyter-scipy image...\n"
$(MAKE) docker-push -C jupyter-scipy
@echo "\nPushing jupyter-pytorch image...\n"
$(MAKE) docker-push-cpu -C jupyter-pytorch
@echo "\nPushing jupyter-pytorch-cuda image...\n"
$(MAKE) docker-push-cuda -C jupyter-pytorch
@echo "\nPushing jupyter-pytorch-full image...\n"
$(MAKE) docker-push-cpu -C jupyter-pytorch-full
@echo "\nPushing jupyter-pytorch-cuda-full image...\n"
$(MAKE) docker-push-cuda -C jupyter-pytorch-full
@echo "\nPushing jupyter-tensorflow image...\n"
$(MAKE) docker-push-cpu -C jupyter-tensorflow
@echo "\nPushing jupyter-tensorflow-cuda image...\n"
$(MAKE) docker-push-cuda -C jupyter-tensorflow
@echo "\nPushing jupyter-tensorflow-full image...\n"
$(MAKE) docker-push-cpu -C jupyter-tensorflow-full
@echo "\nPushing jupyter-tensorflow-cuda-full image...\n"
$(MAKE) docker-push-cuda -C jupyter-tensorflow-full
@echo "\nAll notebook-server images have been successfully pushed...\n"
# ------------------------------------------------------------------------------
# docker-build-push-multi-arch | multi-arch build AND push all images
#
.PHONY: docker-build-push-multi-arch
docker-build-push-multi-arch: $(addprefix docker-build-push-multi-arch--, $(IMAGE_FOLDERS))
docker-build-push-multi-arch--%:
$(MAKE) docker-build-push-multi-arch -C $*

123
components/example-notebook-servers/README.md
View File

@ -1,77 +1,93 @@
# Example Notebook Servers
> 🛑️️ Images are provided as __examples__ and are supported on a best-effort basis.
> These images are provided as __examples__, and are supported on a best-effort basis.
> <br>
> Contributions are greatly appreciated.
## Images
### Images // Base
This chart shows how the images are related to each other (the nodes are clickable links to the Dockerfiles):
```mermaid
graph TD
Base[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/base'>Base</a>] --> Jupyter[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter'>Jupyter</a>]
Base --> Code-Server[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/codeserver'>code-server</a>]
Base --> RStudio[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/rstudio'>RStudio</a>]
Jupyter --> PyTorch[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter-pytorch'>PyTorch</a>]
Jupyter --> SciPy[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter-scipy'>SciPy</a>]
Jupyter --> TensorFlow[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter-tensorflow'>TensorFlow</a>]
Code-Server --> Code-Server-Conda-Python[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/codeserver-python'>Conda Python</a>]
RStudio --> Tidyverse[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/rstudio-tidyverse'>Tidyverse</a>]
PyTorch --> PyTorchFull[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter-pytorch-full'>PyTorch Full</a>]
TensorFlow --> TensorFlowFull[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter-tensorflow-full'>TensorFlow Full</a>]
Jupyter --> PyTorchCuda[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter-pytorch-cuda'>PyTorch CUDA</a>]
Jupyter --> TensorFlowCuda[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter-tensorflow-cuda'>TensorFlow CUDA</a>]
PyTorchCuda --> PyTorchCudaFull[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter-pytorch-cuda-full'>PyTorch CUDA Full</a>]
TensorFlowCuda --> TensorFlowCudaFull[<a href='https://github.com/kubeflow/kubeflow/tree/master/components/example-notebook-servers/jupyter-tensorflow-cuda-full'>TensorFlow CUDA Full</a>]
```
### Base Images
These images provide a common starting point for Kubeflow Notebook containers.
See the [custom images guide](#custom-images) to learn how to extend them with your own packages.
Dockerfile | Registry | Notes
Dockerfile | Container Registry | Notes
--- | --- | ---
[./base](./base) | [`kubeflownotebookswg/base:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/base) | common base image
[./codeserver](./codeserver) | [`kubeflownotebookswg/codeserver:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/codeserver) | base [code-server](https://github.com/cdr/code-server) (Visual Studio Code) image
[./jupyter](./jupyter) | [`kubeflownotebookswg/jupyter:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter) | base [JupyterLab](https://github.com/jupyterlab/jupyterlab) image
[./rstudio](./rstudio) | [`kubeflownotebookswg/rstudio:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/rstudio) | base [RStudio](https://github.com/rstudio/rstudio) image
[`./base`](./base) | [`kubeflownotebookswg/base`](https://hub.docker.com/r/kubeflownotebookswg/base) | Common Base Image
[`./codeserver`](./codeserver) | [`kubeflownotebookswg/codeserver`](https://hub.docker.com/r/kubeflownotebookswg/codeserver) | [code-server](https://github.com/coder/code-server) (Visual Studio Code)
[`./jupyter`](./jupyter) | [`kubeflownotebookswg/jupyter`](https://hub.docker.com/r/kubeflownotebookswg/jupyter) | [JupyterLab](https://github.com/jupyterlab/jupyterlab)
[`./rstudio`](./rstudio) | [`kubeflownotebookswg/rstudio`](https://hub.docker.com/r/kubeflownotebookswg/rstudio) | [RStudio](https://github.com/rstudio/rstudio)
### Images // Full
### Kubeflow Images
These images extend the [base images](#images--base) with common packages used in the real world.
Dockerfile | Registry | Notes
Dockerfile | Container Registry | Notes
--- | --- | ---
[./codeserver-python](./codeserver-python) | [`kubeflownotebookswg/codeserver-python:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/codeserver-python) | code-server (Visual Studio Code) + Conda Python
[./jupyter-pytorch (CPU)](./jupyter-pytorch) | [`kubeflownotebookswg/jupyter-pytorch:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-pytorch) | JupyterLab + PyTorch (CPU)
[./jupyter-pytorch (CUDA)](./jupyter-pytorch) | [`kubeflownotebookswg/jupyter-pytorch-cuda:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-pytorch-cuda) | JupyterLab + PyTorch (CUDA)
[./jupyter-pytorch-full (CPU)](./jupyter-pytorch-full) | [`kubeflownotebookswg/jupyter-pytorch-full:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-pytorch-full) | JupyterLab + PyTorch (CPU) + [common](./jupyter-pytorch-full/requirements.txt) packages
[./jupyter-pytorch-full (CUDA)](./jupyter-pytorch-full) | [`kubeflownotebookswg/jupyter-pytorch-cuda-full:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-pytorch-cuda-full) | JupyterLab + PyTorch (CUDA) + [common](./jupyter-pytorch-full/requirements.txt) packages
[./jupyter-scipy](./jupyter-scipy) | [`kubeflownotebookswg/jupyter-scipy:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-scipy) | JupyterLab + [SciPy](https://www.scipy.org/) packages
[./jupyter-tensorflow (CPU)](./jupyter-tensorflow) | [`kubeflownotebookswg/jupyter-tensorflow:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow) | JupyterLab + TensorFlow (CPU)
[./jupyter-tensorflow (CUDA)](./jupyter-tensorflow) | [`kubeflownotebookswg/jupyter-tensorflow-cuda:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow-cuda) | JupyterLab + TensorFlow (CUDA)
[./jupyter-tensorflow-full (CPU)](./jupyter-tensorflow-full) | [`kubeflownotebookswg/jupyter-tensorflow-full:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow-full) | JupyterLab + TensorFlow (CPU) + [common](./jupyter-tensorflow-full/requirements.txt) packages
[./jupyter-tensorflow-full (CUDA)](./jupyter-tensorflow-full) | [`kubeflownotebookswg/jupyter-tensorflow-cuda-full:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow-cuda-full) | JupyterLab + TensorFlow (CUDA) + [common](./jupyter-tensorflow-full/requirements.txt) packages
[./rstudio-tidyverse](./rstudio-tidyverse) | [`kubeflownotebookswg/rstudio-tidyverse:{TAG}`](https://gallery.ecr.aws/j1r0q0g6/notebooks/notebook-servers/rstudio-tidyverse) | RStudio + [Tidyverse](https://www.tidyverse.org/) packages
### Images // Relationship Chart
This chart shows how the images are related to each other.
![flow-chart of kubeflow notebook server images](image-flow-chart.png)
### Images // Important Information
- make your [custom images](#custom-images) by extending one of the [base images](#images--base)
- images run as the `jovyan` user
- images use the [s6-overlay](https://github.com/just-containers/s6-overlay) init system to manage processes
---
[`./codeserver-python`](./codeserver-python) | [`kubeflownotebookswg/codeserver-python`](https://hub.docker.com/r/kubeflownotebookswg/codeserver-python) | code-server + Conda Python
[`./rstudio-tidyverse`](./rstudio-tidyverse) | [`kubeflownotebookswg/rstudio-tidyverse`](https://hub.docker.com/r/kubeflownotebookswg/rstudio-tidyverse) | RStudio + [Tidyverse](https://www.tidyverse.org/)
[`./jupyter-pytorch`](./jupyter-pytorch) | [`kubeflownotebookswg/jupyter-pytorch`](https://hub.docker.com/r/kubeflownotebookswg/jupyter-pytorch) | JupyterLab + PyTorch
[`./jupyter-pytorch-full`](./jupyter-pytorch-full) | [`kubeflownotebookswg/jupyter-pytorch-full`](https://hub.docker.com/r/kubeflownotebookswg/jupyter-pytorch-full) | JupyterLab + PyTorch + Common Packages
[`./jupyter-pytorch-cuda`](./jupyter-pytorch-cuda) | [`kubeflownotebookswg/jupyter-pytorch-cuda`](https://hub.docker.com/r/kubeflownotebookswg/jupyter-pytorch-cuda) | JupyterLab + PyTorch + CUDA
[`./jupyter-pytorch-cuda-full`](./jupyter-pytorch-cuda-full) | [`kubeflownotebookswg/jupyter-pytorch-cuda-full`](https://hub.docker.com/r/kubeflownotebookswg/jupyter-pytorch-cuda-full) | JupyterLab + PyTorch + CUDA + Common Packages
[`./jupyter-scipy`](./jupyter-scipy) | [`kubeflownotebookswg/jupyter-scipy`](https://hub.docker.com/r/kubeflownotebookswg/jupyter-scipy) | JupyterLab + Common Packages
[`./jupyter-tensorflow`](./jupyter-tensorflow) | [`kubeflownotebookswg/jupyter-tensorflow`](https://hub.docker.com/r/kubeflownotebookswg/jupyter-tensorflow) | JupyterLab + TensorFlow
[`./jupyter-tensorflow-full`](./jupyter-tensorflow-full) | [`kubeflownotebookswg/jupyter-tensorflow-full`](https://hub.docker.com/r/kubeflownotebookswg/jupyter-tensorflow-full) | JupyterLab + TensorFlow + Common Packages
[`./jupyter-tensorflow-cuda`](./jupyter-tensorflow-cuda) | [`kubeflownotebookswg/jupyter-tensorflow-cuda`](https://hub.docker.com/r/kubeflownotebookswg/jupyter-tensorflow-cuda) | JupyterLab + TensorFlow + CUDA
[`./jupyter-tensorflow-cuda-full`](./jupyter-tensorflow-cuda-full) | [`kubeflownotebookswg/jupyter-tensorflow-cuda-full`](https://hub.docker.com/r/kubeflownotebookswg/jupyter-tensorflow-cuda-full) | JupyterLab + TensorFlow + CUDA + Common Packages
## Custom Images
Packages installed by users __after spawning__ a Kubeflow Notebook will only last the lifetime of the pod (unless installed into a PVC-backed directory).
To ensure packages are preserved throughout Pod restarts users will need to either:
1. Build custom images that include them, or
2. Ensure they are installed in a PVC-backed directory
### Custom Images // Python Packages
### Install Python Packages
> ⚠️ a common cause of errors is users running `pip install --user ...`, causing the home-directory (which is backed by a PVC) to contain a different or incompatible version of a package contained in `/opt/conda/...`
You may extend one of the images and install any `pip` or `conda` packages your Kubeflow Notebook users are likely to need.
Extend one of the base images and install any `pip` or `conda` packages your Kubeflow Notebook users are likely to need.
As a guide, look at [`./jupyter-pytorch-full/Dockerfile`](./jupyter-pytorch-full/Dockerfile) for a `pip install ...` example, and the [`./rstudio-tidyverse/Dockerfile`](./rstudio-tidyverse/Dockerfile) for `conda install ...`.
As a guide, look at [jupyter-pytorch-full.cpu](./jupyter-pytorch-full/cpu.Dockerfile) for a `pip install ...` example, and the [rstudio-tidyverse](./rstudio-tidyverse/Dockerfile) for `conda install ...`.
> __NOTE:__
>
> A common cause of errors is users running `pip install --user ...`, causing the home-directory (which is backed by a PVC) to contain a different or incompatible version of a package contained in `/opt/conda/...`
### Install Linux Packages
### Custom Images // Linux Packages
You may extend one of the images and install any `apt-get` packages your Kubeflow Notebook users are likely to need.
> ⚠️ ensure you swap to `root` in the Dockerfile before running `apt-get`, and swap back to `jovyan` after.
> __NOTE:__
>
> Ensure you swap to `root` in the Dockerfile before running `apt-get`, and swap back to `$NB_USER` after.
Extend one of the base images and install any `apt-get` packages your Kubeflow Notebook users are likely to need.
### Custom Images // S6
### Configure S6 Overlay
Some use-cases might require custom scripts to run during the startup of the Notebook Server container, or advanced users might want to add additional services that run inside the container (for example, an Apache or NGINX web server).
To make this easy, we use the [s6-overlay](https://github.com/just-containers/s6-overlay).
@ -79,15 +95,15 @@ To make this easy, we use the [s6-overlay](https://github.com/just-containers/s6
The [s6-overlay](https://github.com/just-containers/s6-overlay) differs from other init systems like [tini](https://github.com/krallin/tini).
While `tini` was created to handle a single process running in a container as PID 1, the `s6-overlay` is built to manage multiple processes and allows the creator of the image to determine which process failures should silently restart, and which should cause the container to exit.
#### Custom Images // S6 // Scripts
#### Create Scripts
Scripts that need to run during the startup of the container can be placed in `/etc/cont-init.d/`, and are executed in ascending alphanumeric order.
An example of a startup script can be found in [./rstudio/s6/cont-init.d/02-rstudio-env-fix](./rstudio/s6/cont-init.d/02-rstudio-env-fix).
An example of a startup script can be found in [`./rstudio/s6/cont-init.d/02-rstudio-env-fix`](./rstudio/s6/cont-init.d/02-rstudio-env-fix).
This script uses the [with-contenv](https://github.com/just-containers/s6-overlay#container-environment) helper so that environment variables (passed to container) are available in the script.
The purpose of this script is to snapshot any `KUBERNETES_*` environment variables into the `Renviron.site` at pod startup, as without these variables `kubectl` does not work.
#### Custom Images // S6 // Services
#### Create Services
Extra services to be monitored by `s6-overlay` should be placed in their own folder under `/etc/services.d/` containing a script called `run` and optionally a finishing script `finish`.
@ -95,19 +111,20 @@ For more information about the `run` and `finish` scripts, please see the [s6-ov
An example of a service can be found in the `run` script of [jupyter/s6/services.d/jupyterlab](jupyter/s6/services.d/jupyterlab) which is used to start JupyterLab itself.
#### Custom Images // S6 // Run As Root
> ⚠️ our example images run `s6-overlay` as `$NB_USER` (not `root`), meaning any files or scripts related to `s6-overlay` must be owned by the `$NB_USER` user to successfully run
#### Run Services As Root
There may be cases when you need to run a service as root, to do this, you can change the Dockerfile to have `USER root` at the end, and then use `s6-setuidgid` to run the user-facing services as `$NB_USER`.
---
> __NOTE:__
>
> Our example images run `s6-overlay` as `$NB_USER` (not `root`), meaning any files or scripts related to `s6-overlay` must be owned by the `$NB_USER` user to successfully run.
## Troubleshooting
### Troubleshooting // Jupyter
### Jupyter
__Kernel stuck in `connecting` state:__
#### Kernel stuck in `connecting` state:
This is a problem that occurs from time to time and is not a Kubeflow problem, but rather a browser.
It can be identified by looking in the browser error console, which will show errors regarding the websocket not connecting.

55
components/example-notebook-servers/base/Dockerfile
View File

@ -1,5 +1,11 @@
# Use the respective Makefile to build the image.
FROM ubuntu:20.04
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<ubuntu>
FROM $BASE_IMG
ARG TARGETARCH
# common environemnt variables
ENV NB_USER jovyan
@ -9,11 +15,8 @@ ENV HOME /home/$NB_USER
ENV SHELL /bin/bash
# args - software versions
ARG KUBECTL_ARCH="amd64"
ARG KUBECTL_VERSION=v1.21.0
ARG S6_ARCH="amd64"
# renovate: datasource=github-tags depName=just-containers/s6-overlay versioning=loose
ARG S6_VERSION=v2.2.0.3
ARG KUBECTL_VERSION=v1.27.6
ARG S6_VERSION=v3.1.5.0
# set shell to bash
SHELL ["/bin/bash", "-c"]
@ -38,24 +41,35 @@ RUN export DEBIAN_FRONTEND=noninteractive \
unzip \
vim \
wget \
xz-utils \
zip \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# install - s6 overlay
RUN export GNUPGHOME=/tmp/ \
&& curl -sL "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-${S6_ARCH}-installer" -o /tmp/s6-overlay-${S6_VERSION}-installer \
&& curl -sL "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-${S6_ARCH}-installer.sig" -o /tmp/s6-overlay-${S6_VERSION}-installer.sig \
&& gpg --keyserver keys.gnupg.net --keyserver pgp.surfnet.nl --recv-keys 6101B2783B2FD161 \
&& gpg -q --verify /tmp/s6-overlay-${S6_VERSION}-installer.sig /tmp/s6-overlay-${S6_VERSION}-installer \
&& chmod +x /tmp/s6-overlay-${S6_VERSION}-installer \
&& /tmp/s6-overlay-${S6_VERSION}-installer / \
&& rm /tmp/s6-overlay-${S6_VERSION}-installer.sig /tmp/s6-overlay-${S6_VERSION}-installer
RUN case "${TARGETARCH}" in \
amd64) S6_ARCH="x86_64" ;; \
arm64) S6_ARCH="aarch64" ;; \
ppc64le) S6_ARCH="ppc64le" ;; \
*) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \
esac \
&& curl -fsSL "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-noarch.tar.xz" -o /tmp/s6-overlay-noarch.tar.xz \
&& curl -fsSL "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-noarch.tar.xz.sha256" -o /tmp/s6-overlay-noarch.tar.xz.sha256 \
&& echo "$(cat /tmp/s6-overlay-noarch.tar.xz.sha256 | awk '{ print $1; }') /tmp/s6-overlay-noarch.tar.xz" | sha256sum -c - \
&& curl -fsSL "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-${S6_ARCH}.tar.xz" -o /tmp/s6-overlay-${S6_ARCH}.tar.xz \
&& curl -fsSL "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-${S6_ARCH}.tar.xz.sha256" -o /tmp/s6-overlay-${S6_ARCH}.tar.xz.sha256 \
&& echo "$(cat /tmp/s6-overlay-${S6_ARCH}.tar.xz.sha256 | awk '{ print $1; }') /tmp/s6-overlay-${S6_ARCH}.tar.xz" | sha256sum -c - \
&& tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz \
&& tar -C / -Jxpf /tmp/s6-overlay-${S6_ARCH}.tar.xz \
&& rm /tmp/s6-overlay-noarch.tar.xz \
/tmp/s6-overlay-noarch.tar.xz.sha256 \
/tmp/s6-overlay-${S6_ARCH}.tar.xz \
/tmp/s6-overlay-${S6_ARCH}.tar.xz.sha256
# install - kubectl
RUN curl -sL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${KUBECTL_ARCH}/kubectl" -o /usr/local/bin/kubectl \
&& curl -sL "https://dl.k8s.io/${KUBECTL_VERSION}/bin/linux/${KUBECTL_ARCH}/kubectl.sha256" -o /tmp/kubectl.sha256 \
&& echo "$(cat /tmp/kubectl.sha256) /usr/local/bin/kubectl" | sha256sum --check \
RUN curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" -o /usr/local/bin/kubectl \
&& curl -fsSL "https://dl.k8s.io/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl.sha256" -o /tmp/kubectl.sha256 \
&& echo "$(cat /tmp/kubectl.sha256 | awk '{ print $1; }') /usr/local/bin/kubectl" | sha256sum -c - \
&& rm /tmp/kubectl.sha256 \
&& chmod +x /usr/local/bin/kubectl
@ -63,8 +77,7 @@ RUN curl -sL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${KUBECTL_A
RUN useradd -M -s /bin/bash -N -u ${NB_UID} ${NB_USER} \
&& mkdir -p ${HOME} \
&& chown -R ${NB_USER}:users ${HOME} \
&& chown -R ${NB_USER}:users /usr/local/bin \
&& chown -R ${NB_USER}:users /etc/s6
&& chown -R ${NB_USER}:users /usr/local/bin
# set locale configs
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
@ -74,3 +87,5 @@ ENV LANGUAGE en_US.UTF-8
ENV LC_ALL en_US.UTF-8
USER $NB_UID
ENTRYPOINT ["/init"]

39
components/example-notebook-servers/base/Makefile
View File

@ -1,8 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build:
docker build -t ${REGISTRY}/base:${TAG} -f Dockerfile .
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-push:
docker push ${REGISTRY}/base:${TAG}
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
IMAGE_NAME := base
BASE_IMAGE := ubuntu:22.04
BASE_IMAGE_FOLDERS :=
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

54
components/example-notebook-servers/codeserver-python/Dockerfile
View File

@ -1,29 +1,43 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<codeserver>
FROM $BASE_IMG
ARG TARGETARCH
USER root
# args - software versions
ARG CODESERVER_PYTHON_VERSION=2021.5.842923320
ARG MINIFORGE_ARCH="x86_64"
# renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
ARG MINIFORGE_VERSION=4.10.1-4
ARG PIP_VERSION=21.1.2
ARG PYTHON_VERSION=3.8.10
ARG CODESERVER_PYTHON_VERSION=2023.18.0
ARG MINIFORGE_VERSION=23.3.1-1
ARG PIP_VERSION=23.2.1
ARG PYTHON_VERSION=3.11.6
# setup environment for conda
ENV CONDA_DIR /opt/conda
ENV PATH "${CONDA_DIR}/bin:${PATH}"
RUN mkdir -p ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${CONDA_DIR}
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> ${HOME}/.bashrc \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> /etc/profile \
&& echo "conda activate base" >> ${HOME}/.bashrc \
&& echo "conda activate base" >> /etc/profile \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
USER $NB_UID
# install - conda, pip, python
RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh" -o /tmp/Miniforge3.sh \
&& curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh.sha256" -o /tmp/Miniforge3.sh.sha256 \
&& echo "$(cat /tmp/Miniforge3.sh.sha256 | awk '{ print $1; }') /tmp/Miniforge3.sh" | sha256sum --check \
RUN case "${TARGETARCH}" in \
amd64) MINIFORGE_ARCH="x86_64" ;; \
arm64) MINIFORGE_ARCH="aarch64" ;; \
ppc64le) MINIFORGE_ARCH="ppc64le" ;; \
*) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \
esac \
&& curl -fsSL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh" -o /tmp/Miniforge3.sh \
&& curl -fsSL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh.sha256" -o /tmp/Miniforge3.sh.sha256 \
&& echo "$(cat /tmp/Miniforge3.sh.sha256 | awk '{ print $1; }') /tmp/Miniforge3.sh" | sha256sum -c - \
&& rm /tmp/Miniforge3.sh.sha256 \
&& /bin/bash /tmp/Miniforge3.sh -b -f -p ${CONDA_DIR} \
&& rm /tmp/Miniforge3.sh \
@ -36,28 +50,20 @@ RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIF
conda=${MINIFORGE_VERSION:0:-2} \
pip=${PIP_VERSION} \
&& conda update -y -q --all \
&& conda clean -a -f -y \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
&& conda clean -a -f -y
# install - requirements.txt
COPY --chown=jovyan:users requirements.txt /tmp
COPY --chown=${NB_USER}:users requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
&& rm -f /tmp/requirements.txt
# install - codeserver extensions
RUN curl -# -L -o /tmp/ms-python-release.vsix "https://github.com/microsoft/vscode-python/releases/download/${CODESERVER_PYTHON_VERSION}/ms-python-release.vsix" \
&& code-server --install-extension /tmp/ms-python-release.vsix \
RUN code-server --install-extension "ms-python.python@${CODESERVER_PYTHON_VERSION}" --force \
&& code-server --list-extensions --show-versions
# s6 - copy scripts
COPY --chown=jovyan:users s6/ /etc
# s6 - 01-copy-tmp-home
USER root
RUN mkdir -p /tmp_home \
&& cp -r ${HOME} /tmp_home \
&& chown -R ${NB_USER}:users /tmp_home
USER ${NB_UID}
USER $NB_UID

40
components/example-notebook-servers/codeserver-python/Makefile
View File

@ -1,11 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-codeserver:
$(MAKE) docker-build -C ../codeserver TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build: docker-build-codeserver
docker build -t ${REGISTRY}/codeserver-python:${TAG} --build-arg BASE_IMG=${REGISTRY}/codeserver:${TAG} -f Dockerfile .
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-push:
docker push ${REGISTRY}/codeserver-python:${TAG}
IMAGE_NAME := codeserver-python
BASE_IMAGE := $(REGISTRY)/codeserver:$(TAG)
BASE_IMAGE_FOLDERS := codeserver
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

6
components/example-notebook-servers/codeserver-python/requirements.txt
View File

@ -1,3 +1,3 @@
kfp==1.6.3
kfp-server-api==1.6.0
kfserving==0.5.1
# kubeflow packages
kfp==2.3.0
kfp-server-api==2.0.2

3
components/example-notebook-servers/codeserver-python/s6/cont-init.d/02-conda-init
View File

@ -1,3 +0,0 @@
#!/usr/bin/with-contenv bash
conda init bash
conda activate base

20
components/example-notebook-servers/codeserver/Dockerfile
View File

@ -1,20 +1,24 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<base>
FROM $BASE_IMG
USER root
ARG TARGETARCH
# args - software versions
# renovate: datasource=github-tags depName=cdr/code-server versioning=semver
ARG CODESERVER_VERSION=v4.3.0
ARG CODESERVER_VERSION=v4.17.1
USER root
# install - code-server
RUN curl -sL "https://github.com/cdr/code-server/releases/download/${CODESERVER_VERSION}/code-server_${CODESERVER_VERSION/v/}_amd64.deb" -o /tmp/code-server.deb \
RUN curl -fsSL "https://github.com/coder/code-server/releases/download/${CODESERVER_VERSION}/code-server_${CODESERVER_VERSION/v/}_${TARGETARCH}.deb" -o /tmp/code-server.deb \
&& dpkg -i /tmp/code-server.deb \
&& rm -f /tmp/code-server.deb
# s6 - copy scripts
COPY --chown=jovyan:users s6/ /etc
COPY --chown=${NB_USER}:users --chmod=755 s6/ /etc
# s6 - 01-copy-tmp-home
RUN mkdir -p /tmp_home \
@ -23,6 +27,4 @@ RUN mkdir -p /tmp_home \
USER $NB_UID
EXPOSE 8888
ENTRYPOINT ["/init"]
EXPOSE 8888

40
components/example-notebook-servers/codeserver/Makefile
View File

@ -1,11 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-base:
$(MAKE) docker-build -C ../base TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build: docker-build-base
docker build -t ${REGISTRY}/codeserver:${TAG} --build-arg BASE_IMG=${REGISTRY}/base:${TAG} -f Dockerfile .
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-push:
docker push ${REGISTRY}/codeserver:${TAG}
IMAGE_NAME := codeserver
BASE_IMAGE := $(REGISTRY)/base:$(TAG)
BASE_IMAGE_FOLDERS := base
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

2
components/example-notebook-servers/codeserver/s6/cont-init.d/01-copy-tmp-home
View File

@ -1,2 +1,2 @@
#!/usr/bin/with-contenv bash
#!/command/with-contenv bash
cp -r -n /tmp_home/* /home/

9
components/example-notebook-servers/codeserver/s6/services.d/code-server/run
View File

@ -1,5 +1,10 @@
#!/usr/bin/with-contenv bash
#!/command/with-contenv bash
cd "${HOME}"
exec /usr/bin/code-server \
--bind-addr 0.0.0.0:8888 \
--disable-telemetry \
--auth none
--disable-update-check \
--disable-workspace-trust \
--disable-getting-started-override \
--auth none \
"${HOME}"

114
components/example-notebook-servers/common.mk Normal file
View File

@ -0,0 +1,114 @@
#
# This partial Makefile contains common targets for building and pushing docker images.
#
# It expects the following variables to be set:
# - REGISTRY: the container registry to push to
# - TAG: the primary tag to use
# - IMAGE_NAME: name of the image to build
# - BASE_IMAGE: the base image to use
# - BASE_IMAGE_FOLDERS: the folders containing the base images (whitespace separated)
# - ARCH: the buildkit platforms to build for
# - CACHE_IMAGE: an image to use as a registry-type cache
# - CACHE_TAG: the tag to use for the cache image
#
ALSO_TAG_LATEST ?= 0
ALSO_TAG_VERSION ?=
# compute image ref
IMAGE_REF := $(REGISTRY)/$(IMAGE_NAME)
# compute tag args
IMAGE_TAGS := $(TAG)
ifeq ($(ALSO_TAG_LATEST),1)
IMAGE_TAGS += latest
endif
ifneq ($(ALSO_TAG_VERSION),)
IMAGE_TAGS += $(ALSO_TAG_VERSION)
endif
TAG_ARGS := $(foreach tag_val,$(IMAGE_TAGS),--tag "$(IMAGE_REF):$(tag_val)")
TAG_NAMES := $(foreach tag_val,$(IMAGE_TAGS),"$(IMAGE_REF):$(tag_val)")
# compute cache args
CACHE_FROM := type=registry,ref=$(CACHE_IMAGE):$(CACHE_TAG)
CACHE_TO := type=registry,ref=$(CACHE_IMAGE):$(CACHE_TAG),mode=max
# ------------------------------------------------------------------------------
# docker-build
# ------------------------------------------------------------------------------
# build images in this folder
.PHONY: docker-build
docker-build:
@echo "\n\n"
@echo "------------------------------------------------------------------------------"
@echo "Building '$(IMAGE_NAME)' image..."
@echo "------------------------------------------------------------------------------"
@echo ""
docker build --build-arg BASE_IMG=$(BASE_IMAGE) --progress=plain $(TAG_ARGS) -f Dockerfile .
# build base images as well
.PHONY: docker-build-dep
docker-build-dep: $(addprefix docker-build-dep--, $(BASE_IMAGE_FOLDERS)) docker-build
docker-build-dep--%:
$(MAKE) docker-build-dep -C ../$*
# ------------------------------------------------------------------------------
# docker-push
# ------------------------------------------------------------------------------
# push images in this folder
.PHONY: docker-push
docker-push:
@echo "\n\n"
@echo "------------------------------------------------------------------------------"
@echo "Pushing '$(IMAGE_NAME)' image..."
@echo "------------------------------------------------------------------------------"
@echo ""
docker push $(TAG_NAMES)
# push base images as well
.PHONY: docker-push-dep
docker-push-dep: $(addprefix docker-push-dep--, $(BASE_IMAGE_FOLDERS)) docker-push
docker-push-dep--%:
$(MAKE) docker-push-dep -C ../$*
# ------------------------------------------------------------------------------
# docker-build-multi-arch
# ------------------------------------------------------------------------------
# multi-arch build images in this folder
.PHONY: docker-build-multi-arch
docker-build-multi-arch:
@echo "\n\n"
@echo "------------------------------------------------------------------------------"
@echo "Building '$(IMAGE_NAME)' image for '$(ARCH)'..."
@echo "------------------------------------------------------------------------------"
@echo ""
docker buildx build --load --platform $(ARCH) --build-arg BASE_IMG=$(BASE_IMAGE) --progress=plain --cache-from $(CACHE_FROM) $(TAG_ARGS) -f Dockerfile .
# multi-arch build base images as well
.PHONY: docker-build-multi-arch-dep
docker-build-multi-arch-dep: $(addprefix docker-build-multi-arch-dep--, $(BASE_IMAGE_FOLDERS)) docker-build-multi-arch
docker-build-multi-arch-dep--%:
$(MAKE) docker-build-multi-arch-dep -C ../$*
# ------------------------------------------------------------------------------
# docker-build-push-multi-arch
# ------------------------------------------------------------------------------
# multi-arch build AND push images in this folder
.PHONY: docker-build-push-multi-arch
docker-build-push-multi-arch:
@echo "\n\n"
@echo "------------------------------------------------------------------------------"
@echo "Building AND Pushing '$(IMAGE_NAME)' image for '$(ARCH)'..."
@echo "------------------------------------------------------------------------------"
@echo ""
docker buildx build --push --platform $(ARCH) --build-arg BASE_IMG=$(BASE_IMAGE) --progress=plain --cache-from $(CACHE_FROM) --cache-to $(CACHE_TO) $(TAG_ARGS) -f Dockerfile .
# multi-arch build AND push base images as well
.PHONY: docker-build-push-multi-arch-dep
docker-build-push-multi-arch-dep: $(addprefix docker-build-push-multi-arch-dep--, $(BASE_IMAGE_FOLDERS)) docker-build-push-multi-arch
docker-build-push-multi-arch-dep--%:
$(MAKE) docker-build-push-multi-arch-dep -C ../$*

27
components/example-notebook-servers/jupyter-pytorch-cuda-full/Dockerfile Normal file
View File

@ -0,0 +1,27 @@
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<jupyter-pytorch-cuda>
FROM $BASE_IMG
# install - conda packages
# NOTE: we use mamba to speed things up
RUN mamba install -y -q \
bokeh==3.2.2 \
cloudpickle==2.2.1 \
dill==0.3.7 \
ipympl==0.9.3 \
matplotlib==3.8.0 \
pandas==2.1.1 \
scikit-image==0.22.0 \
scikit-learn==1.3.1 \
scipy==1.11.3 \
seaborn==0.13.0 \
xgboost==1.7.6 \
&& mamba clean -a -f -y
# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

35
components/example-notebook-servers/jupyter-pytorch-cuda-full/Makefile Normal file
View File

@ -0,0 +1,35 @@
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
IMAGE_NAME := jupyter-pytorch-cuda-full
BASE_IMAGE := $(REGISTRY)/jupyter-pytorch-cuda:$(TAG)
BASE_IMAGE_FOLDERS := jupyter-pytorch-cuda
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

3
components/example-notebook-servers/jupyter-pytorch-cuda-full/requirements.txt Normal file
View File

@ -0,0 +1,3 @@
# kubeflow packages
kfp==2.3.0
kfp-server-api==2.0.2

28
components/example-notebook-servers/jupyter-pytorch-cuda/Dockerfile Normal file
View File

@ -0,0 +1,28 @@
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<jupyter>
FROM $BASE_IMG
# args - software versions
ARG PYTORCH_VERSION=2.1.0
ARG TORCHAUDIO_VERSION=2.1.0
ARG TORCHVISION_VERSION=0.16.0
# nvidia container toolkit
# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV NVIDIA_REQUIRE_CUDA "cuda>=12.1"
# install - pytorch (cuda)
RUN python3 -m pip install --quiet --no-cache-dir --index-url https://download.pytorch.org/whl/cu121 \
torch==${PYTORCH_VERSION} \
torchvision==${TORCHVISION_VERSION} \
torchaudio==${TORCHAUDIO_VERSION}
# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

35
components/example-notebook-servers/jupyter-pytorch-cuda/Makefile Normal file
View File

@ -0,0 +1,35 @@
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
IMAGE_NAME := jupyter-pytorch-cuda
BASE_IMAGE := $(REGISTRY)/jupyter:$(TAG)
BASE_IMAGE_FOLDERS := jupyter
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

0
components/example-notebook-servers/jupyter-pytorch-cuda/requirements.txt Normal file
View File

27
components/example-notebook-servers/jupyter-pytorch-full/Dockerfile Normal file
View File

@ -0,0 +1,27 @@
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<jupyter-pytorch>
FROM $BASE_IMG
# install - conda packages
# NOTE: we use mamba to speed things up
RUN mamba install -y -q \
bokeh==3.2.2 \
cloudpickle==2.2.1 \
dill==0.3.7 \
ipympl==0.9.3 \
matplotlib==3.8.0 \
pandas==2.1.1 \
scikit-image==0.22.0 \
scikit-learn==1.3.1 \
scipy==1.11.3 \
seaborn==0.13.0 \
xgboost==1.7.6 \
&& mamba clean -a -f -y
# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

43
components/example-notebook-servers/jupyter-pytorch-full/Makefile
View File

@ -1,20 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-jupyter-pytorch:
$(MAKE) docker-build-cpu -C ../jupyter-pytorch TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build-jupyter-pytorch-cuda:
$(MAKE) docker-build-cuda -C ../jupyter-pytorch TAG=${TAG}
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-build-cpu: docker-build-jupyter-pytorch
docker build -t ${REGISTRY}/jupyter-pytorch-full:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter-pytorch:${TAG} -f cpu.Dockerfile .
IMAGE_NAME := jupyter-pytorch-full
docker-build-cuda: docker-build-jupyter-pytorch-cuda
docker build -t ${REGISTRY}/jupyter-pytorch-cuda-full:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter-pytorch-cuda:${TAG} -f cuda.Dockerfile .
BASE_IMAGE := $(REGISTRY)/jupyter-pytorch:$(TAG)
BASE_IMAGE_FOLDERS := jupyter-pytorch
docker-push-cpu:
docker push ${REGISTRY}/jupyter-pytorch-full:${TAG}
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
docker-push-cuda:
docker push ${REGISTRY}/jupyter-pytorch-cuda-full:${TAG}
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

8
components/example-notebook-servers/jupyter-pytorch-full/cpu.Dockerfile
View File

@ -1,8 +0,0 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
ARG BASE_IMG=<jupyter-pytorch>
FROM $BASE_IMG
# install - requirements.txt
COPY --chown=jovyan:users requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

8
components/example-notebook-servers/jupyter-pytorch-full/cuda.Dockerfile
View File

@ -1,8 +0,0 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
ARG BASE_IMG=<jupyter-pytorch-cuda>
FROM $BASE_IMG
# install - requirements.txt
COPY --chown=jovyan:users requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

24
components/example-notebook-servers/jupyter-pytorch-full/requirements.txt
View File

@ -1,23 +1,3 @@
# kubeflow packages
kfp==1.6.3
kfp-server-api==1.6.0
kfserving==0.5.1
# common packages
bokeh==2.3.2
cloudpickle==1.6.0
dill==0.3.4
ipympl==0.7.0
ipywidgets==7.6.3
jupyterlab-git==0.30.1
matplotlib==3.4.2
pandas==1.2.4
scikit-image==0.18.1
scikit-learn==0.24.2
scipy==1.7.0
seaborn==0.11.1
xgboost==1.4.2
# pytorch packages
#torchelastic==0.2.2 this currently causes a dependency conflict, should be fixed very soon
fastai==2.4
kfp==2.3.0
kfp-server-api==2.0.2

22
components/example-notebook-servers/jupyter-pytorch/Dockerfile Normal file
View File

@ -0,0 +1,22 @@
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<jupyter>
FROM $BASE_IMG
# args - software versions
ARG PYTORCH_VERSION=2.1.0
ARG TORCHAUDIO_VERSION=2.1.0
ARG TORCHVISION_VERSION=0.16.0
# install - pytorch (cpu)
RUN python3 -m pip install --quiet --no-cache-dir --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \
torch==${PYTORCH_VERSION} \
torchvision==${TORCHVISION_VERSION} \
torchaudio==${TORCHAUDIO_VERSION}
# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

42
components/example-notebook-servers/jupyter-pytorch/Makefile
View File

@ -1,17 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-jupyter:
$(MAKE) docker-build -C ../jupyter TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build-cpu: docker-build-jupyter
docker build -t ${REGISTRY}/jupyter-pytorch:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter:${TAG} -f cpu.Dockerfile .
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-build-cuda: docker-build-jupyter
docker build -t ${REGISTRY}/jupyter-pytorch-cuda:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter:${TAG} -f cuda.Dockerfile .
IMAGE_NAME := jupyter-pytorch
docker-push-cpu:
docker push ${REGISTRY}/jupyter-pytorch:${TAG}
BASE_IMAGE := $(REGISTRY)/jupyter:$(TAG)
BASE_IMAGE_FOLDERS := jupyter
docker-push-cuda:
docker push ${REGISTRY}/jupyter-pytorch-cuda:${TAG}
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

4
components/example-notebook-servers/jupyter-pytorch/cpu-requirements.txt
View File

@ -1,4 +0,0 @@
--find-links https://download.pytorch.org/whl/torch_stable.html
torch==1.8.1+cpu
torchvision==0.9.1+cpu
torchaudio==0.8.1

8
components/example-notebook-servers/jupyter-pytorch/cpu.Dockerfile
View File

@ -1,8 +0,0 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
ARG BASE_IMG=<jupyter>
FROM $BASE_IMG
# install - requirements.txt
COPY --chown=jovyan:users cpu-requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

4
components/example-notebook-servers/jupyter-pytorch/cuda-requirements.txt
View File

@ -1,4 +0,0 @@
--find-links https://download.pytorch.org/whl/torch_stable.html
torch==1.8.1+cu111
torchvision==0.9.1+cu111
torchaudio==0.8.1

13
components/example-notebook-servers/jupyter-pytorch/cuda.Dockerfile
View File

@ -1,13 +0,0 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
ARG BASE_IMG=<jupyter>
FROM $BASE_IMG
# nvidia configs
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
# install - requirements.txt
COPY --chown=jovyan:users cuda-requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

0
components/example-notebook-servers/jupyter-pytorch/requirements.txt Normal file
View File

55
components/example-notebook-servers/jupyter-scipy/Dockerfile
View File

@ -1,8 +1,57 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<jupyter>
FROM $BASE_IMG
# install - usefull linux packages
USER root
RUN apt-get -yq update \
&& apt-get -yq install --no-install-recommends \
# for latex
cm-super \
dvipng \
# for matplotlib
ffmpeg \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
USER $NB_UID
# install - conda packages
# NOTE: we use mamba to speed things up
RUN mamba install -y -q \
altair \
beautifulsoup4==4.12.2 \
bokeh==3.3.0 \
bottleneck \
brotli \
cloudpickle \
dask==2023.10.0 \
dill \
h5py \
ipympl \
matplotlib==3.8.0 \
numba \
numexpr \
openblas==0.3.24 \
openpyxl \
pandas==2.1.1 \
patsy \
protobuf \
pytables==3.9.1 \
scikit-image==0.22.0 \
scikit-learn==1.3.1 \
scipy==1.11.3 \
seaborn==0.13.0 \
sqlalchemy==2.0.22 \
statsmodels \
sympy \
vincent \
xlrd \
&& mamba clean -a -f -y
# install - requirements.txt
COPY --chown=jovyan:users requirements.txt /tmp/requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
&& rm -f /tmp/requirements.txt

40
components/example-notebook-servers/jupyter-scipy/Makefile
View File

@ -1,11 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-jupyter:
$(MAKE) docker-build -C ../jupyter TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build: docker-build-jupyter
docker build -t ${REGISTRY}/jupyter-scipy:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter:${TAG} -f Dockerfile .
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-push:
docker push ${REGISTRY}/jupyter-scipy:${TAG}
IMAGE_NAME := jupyter-scipy
BASE_IMAGE := $(REGISTRY)/jupyter:$(TAG)
BASE_IMAGE_FOLDERS := jupyter
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

35
components/example-notebook-servers/jupyter-scipy/requirements.txt
View File

@ -1,34 +1,3 @@
# kubeflow packages
kfp==1.6.3
kfp-server-api==1.6.0
kfserving==0.5.1
# scipy packages
# https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/Dockerfile
beautifulsoup4==4.9.3
bokeh==2.3.2
#Bottleneck==1.3.2 Could not build wheels for Bottleneck which use PEP 517 and cannot be installed directly
cloudpickle==1.6.0
cython==0.29.23
dask==2021.6.1
dill==0.3.4
h5py==3.2.1
ipympl==0.7.0
ipywidgets==7.6.3
jupyterlab-git==0.30.1
matplotlib==3.4.2
numba==0.53.1
numexpr==2.7.3
pandas==1.2.4
patsy==0.5.1
protobuf==3.17.3
scikit-image==0.18.1
scikit-learn==0.24.2
scipy==1.7.0
seaborn==0.11.1
SQLAlchemy==1.4.18
statsmodels==0.12.2
sympy==1.8
tables==3.6.1
vincent==0.4.4
xlrd==2.0.1
kfp==2.3.0
kfp-server-api==2.0.2

28
components/example-notebook-servers/jupyter-tensorflow-cuda-full/Dockerfile Normal file
View File

@ -0,0 +1,28 @@
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<jupyter-tensorflow-cuda>
FROM $BASE_IMG
# install - conda packages
# NOTE: we use mamba to speed things up
RUN mamba install -y -q \
bokeh==3.2.2 \
cloudpickle==2.2.1 \
dill==0.3.7 \
ipympl==0.9.3 \
matplotlib==3.8.0 \
numpy==1.24.3 \
pandas==2.1.1 \
scikit-image==0.22.0 \
scikit-learn==1.3.1 \
scipy==1.11.3 \
seaborn==0.13.0 \
xgboost==1.7.6 \
&& mamba clean -a -f -y
# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

35
components/example-notebook-servers/jupyter-tensorflow-cuda-full/Makefile Normal file
View File

@ -0,0 +1,35 @@
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
IMAGE_NAME := jupyter-tensorflow-cuda-full
BASE_IMAGE := $(REGISTRY)/jupyter-tensorflow-cuda:$(TAG)
BASE_IMAGE_FOLDERS := jupyter-tensorflow-cuda
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

3
components/example-notebook-servers/jupyter-tensorflow-cuda-full/requirements.txt Normal file
View File

@ -0,0 +1,3 @@
# kubeflow packages
kfp==2.3.0
kfp-server-api==2.0.2

69
components/example-notebook-servers/jupyter-tensorflow-cuda/Dockerfile Normal file
View File

@ -0,0 +1,69 @@
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<jupyter>
FROM $BASE_IMG
ARG TARGETARCH
# args - software versions
ARG TENSORFLOW_VERSION=2.13.0
# args - cuda versions
# - TF CUDA version matrix: https://www.tensorflow.org/install/source#gpu
# - CUDA repo (for package version numbers): https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/
ARG CUDA_VERSION=11.8
ARG CUDNN_VERSION=8.6.0.163
ARG TENSORRT_VERSION=8.6.1.6
USER root
# install - cuda
RUN case "${TARGETARCH}" in \
amd64) CUDA_ARCH="x86_64" ;; \
arm64) CUDA_ARCH="sbsa" ;; \
*) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \
esac \
&& curl -sL "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/${CUDA_ARCH}/3bf863cc.pub" | apt-key add - \
&& echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/${CUDA_ARCH}/ /" > /etc/apt/sources.list.d/cuda.list \
&& apt-get -yq update \
&& apt-get -yq install --no-install-recommends --no-install-suggests \
cuda-command-line-tools-${CUDA_VERSION/./-} \
cuda-compat-${CUDA_VERSION/./-} \
cuda-cudart-${CUDA_VERSION/./-} \
cuda-libraries-${CUDA_VERSION/./-} \
cuda-nvcc-${CUDA_VERSION/./-} \
libcudnn8=${CUDNN_VERSION}-1+cuda${CUDA_VERSION} \
libnvinfer8=${TENSORRT_VERSION}-1+cuda${CUDA_VERSION} \
libnvinfer-plugin8=${TENSORRT_VERSION}-1+cuda${CUDA_VERSION} \
&& apt-mark hold \
libcublas-${CUDA_VERSION/./-} \
libnccl2 \
libcudnn8 \
libnvinfer8 \
libnvinfer-plugin8 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& ln -s /usr/local/cuda-${CUDA_VERSION} /usr/local/cuda
USER $NB_UID
# envs - cuda
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
# nvidia container toolkit
# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV NVIDIA_REQUIRE_CUDA "cuda>=${CUDA_VERSION}"
# install - tensorflow
RUN python3 -m pip install --quiet --no-cache-dir \
tensorflow==${TENSORFLOW_VERSION}
# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

35
components/example-notebook-servers/jupyter-tensorflow-cuda/Makefile Normal file
View File

@ -0,0 +1,35 @@
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
IMAGE_NAME := jupyter-tensorflow-cuda
BASE_IMAGE := $(REGISTRY)/jupyter:$(TAG)
BASE_IMAGE_FOLDERS := jupyter
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

0
components/example-notebook-servers/jupyter-tensorflow-cuda/requirements.txt Normal file
View File

28
components/example-notebook-servers/jupyter-tensorflow-full/Dockerfile Normal file
View File

@ -0,0 +1,28 @@
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<jupyter-tensorflow>
FROM $BASE_IMG
# install - conda packages
# NOTE: we use mamba to speed things up
RUN mamba install -y -q \
bokeh==3.2.2 \
cloudpickle==2.2.1 \
dill==0.3.7 \
ipympl==0.9.3 \
matplotlib==3.8.0 \
numpy==1.24.3 \
pandas==2.1.1 \
scikit-image==0.22.0 \
scikit-learn==1.3.1 \
scipy==1.11.3 \
seaborn==0.13.0 \
xgboost==1.7.6 \
&& mamba clean -a -f -y
# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

43
components/example-notebook-servers/jupyter-tensorflow-full/Makefile
View File

@ -1,20 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-jupyter-tensorflow:
$(MAKE) docker-build-cpu -C ../jupyter-tensorflow TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build-jupyter-tensorflow-cuda:
$(MAKE) docker-build-cuda -C ../jupyter-tensorflow TAG=${TAG}
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-build-cpu: docker-build-jupyter-tensorflow
docker build -t ${REGISTRY}/jupyter-tensorflow-full:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter-tensorflow:${TAG} -f cpu.Dockerfile .
IMAGE_NAME := jupyter-tensorflow-full
docker-build-cuda: docker-build-jupyter-tensorflow-cuda
docker build -t ${REGISTRY}/jupyter-tensorflow-cuda-full:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter-tensorflow-cuda:${TAG} -f cuda.Dockerfile .
BASE_IMAGE := $(REGISTRY)/jupyter-tensorflow:$(TAG)
BASE_IMAGE_FOLDERS := jupyter-tensorflow
docker-push-cpu:
docker push ${REGISTRY}/jupyter-tensorflow-full:${TAG}
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
docker-push-cuda:
docker push ${REGISTRY}/jupyter-tensorflow-cuda-full:${TAG}
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

8
components/example-notebook-servers/jupyter-tensorflow-full/cpu.Dockerfile
View File

@ -1,8 +0,0 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
ARG BASE_IMG=<jupyter-tensorflow>
FROM $BASE_IMG
# install - requirements.txt
COPY --chown=jovyan:users requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

7
components/example-notebook-servers/jupyter-tensorflow-full/cuda.Dockerfile
View File

@ -1,7 +0,0 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
ARG BASE_IMG=<jupyter-tensorflow-cuda>
FROM $BASE_IMG
# install - requirements.txt
COPY --chown=jovyan:users requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

20
components/example-notebook-servers/jupyter-tensorflow-full/requirements.txt
View File

@ -1,19 +1,3 @@
# kubeflow packages
kfp==1.6.3
kfp-server-api==1.6.0
kfserving==0.5.1
# common packages
bokeh==2.3.2
cloudpickle==1.6.0
dill==0.3.4
ipympl==0.7.0
ipywidgets==7.6.3
jupyterlab-git==0.30.1
matplotlib==3.4.2
pandas==1.2.4
scikit-image==0.18.1
scikit-learn==0.24.2
scipy==1.7.0
seaborn==0.11.1
xgboost==1.4.2
kfp==2.3.0
kfp-server-api==2.0.2

18
components/example-notebook-servers/jupyter-tensorflow/Dockerfile Normal file
View File

@ -0,0 +1,18 @@
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<jupyter>
FROM $BASE_IMG
# args - software versions
ARG TENSORFLOW_VERSION=2.13.0
# install - tensorflow
RUN python3 -m pip install --quiet --no-cache-dir \
tensorflow==${TENSORFLOW_VERSION}
# install - requirements.txt
COPY --chown=${NB_USER}:users requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

42
components/example-notebook-servers/jupyter-tensorflow/Makefile
View File

@ -1,17 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-jupyter:
$(MAKE) docker-build -C ../jupyter TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build-cpu: docker-build-jupyter
docker build -t ${REGISTRY}/jupyter-tensorflow:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter:${TAG} -f cpu.Dockerfile .
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-build-cuda: docker-build-jupyter
docker build -t ${REGISTRY}/jupyter-tensorflow-cuda:${TAG} --build-arg BASE_IMG=${REGISTRY}/jupyter:${TAG} -f cuda.Dockerfile .
IMAGE_NAME := jupyter-tensorflow
docker-push-cpu:
docker push ${REGISTRY}/jupyter-tensorflow:${TAG}
BASE_IMAGE := $(REGISTRY)/jupyter:$(TAG)
BASE_IMAGE_FOLDERS := jupyter
docker-push-cuda:
docker push ${REGISTRY}/jupyter-tensorflow-cuda:${TAG}
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

1
components/example-notebook-servers/jupyter-tensorflow/cpu-requirements.txt
View File

@ -1 +0,0 @@
tensorflow==2.9.3

8
components/example-notebook-servers/jupyter-tensorflow/cpu.Dockerfile
View File

@ -1,8 +0,0 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
ARG BASE_IMG=<jupyter>
FROM $BASE_IMG
# install - requirements.txt
COPY --chown=jovyan:users cpu-requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

1
components/example-notebook-servers/jupyter-tensorflow/cuda-requirements.txt
View File

@ -1 +0,0 @@
tensorflow-gpu==2.9.3

84
components/example-notebook-servers/jupyter-tensorflow/cuda.Dockerfile
View File

@ -1,84 +0,0 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
ARG BASE_IMG=<jupyter>
FROM $BASE_IMG
USER root
# needed for LIBNVINFER
ARG OLD_CUDA_VERSION=11.1
# args - software versions
ARG CUDA_VERSION=11.2
ARG CUDA_COMPAT_VERSION=460.73.01-1
ARG CUDA_CUDART_VERSION=11.2.152-1
ARG CUDNN_VERSION=8.1.0.77-1
ARG LIBNVINFER_VERSION=7.2.3-1
# we need bash's env var character substitution
SHELL ["/bin/bash", "-c"]
# install - cuda
# for `cuda-compat-*`: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
RUN curl -sL "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub" | apt-key add - \
&& echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" > /etc/apt/sources.list.d/cuda.list \
&& apt-get -yq update \
&& apt-get -yq install --no-install-recommends \
cuda-compat-${CUDA_VERSION/./-}=${CUDA_COMPAT_VERSION} \
cuda-cudart-${CUDA_VERSION/./-}=${CUDA_CUDART_VERSION} \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& ln -s /usr/local/cuda-${CUDA_VERSION} /usr/local/cuda
# envs - cuda
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV NVIDIA_REQUIRE_CUDA "cuda>=${CUDA_VERSION}"
# install - other nvidia stuff
RUN curl -sL "https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64/7fa2af80.pub" | apt-key add - \
&& echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list \
&& echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list \
&& apt-get -yq update \
&& apt-get -yq install --no-install-recommends \
cm-super \
cuda-command-line-tools-${CUDA_VERSION/./-} \
cuda-nvrtc-${CUDA_VERSION/./-} \
libcublas-${CUDA_VERSION/./-} \
libcudnn8=${CUDNN_VERSION}+cuda${CUDA_VERSION} \
libcufft-${CUDA_VERSION/./-} \
libcurand-${CUDA_VERSION/./-} \
libcusolver-${CUDA_VERSION/./-} \
libcusparse-${CUDA_VERSION/./-} \
libfreetype6-dev \
libhdf5-serial-dev \
libnvinfer7=${LIBNVINFER_VERSION}+cuda${OLD_CUDA_VERSION} \
libnvinfer-plugin7=${LIBNVINFER_VERSION}+cuda${OLD_CUDA_VERSION} \
libzmq3-dev \
pkg-config \
# can't be used until NVIDIA updates (requires python < 3.7)
# python3-libnvinfer=${LIBNVINFER_VERSION}+cuda${CUDA_VERSION} \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# tensorflow fix - CUDA profiling, tensorflow requires CUPTI
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
# tensorflow fix - wrong libcuda lib path (+ reconfigure dynamic linker run-time bindings)
RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \
&& echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \
&& ldconfig
# tensorflow fix - wrong libcusolver lib path
# https://github.com/tensorflow/tensorflow/issues/43947#issuecomment-748273679
RUN ln -s /usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib/libcusolver.so.11 /usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib/libcusolver.so.10
# tensorflow fix - some tensorflow tools expect a `python` binary
RUN ln -s $(which python3) /usr/local/bin/python
USER $NB_UID
# install - requirements.txt
COPY --chown=jovyan:users cuda-requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

0
components/example-notebook-servers/jupyter-tensorflow/requirements.txt Normal file
View File

69
components/example-notebook-servers/jupyter/Dockerfile
View File

@ -1,20 +1,26 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<base>
FROM $BASE_IMG
ARG TARGETARCH
USER root
# args - software versions
ARG MINIFORGE_ARCH="x86_64"
# renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
ARG MINIFORGE_VERSION=4.10.1-4
ARG PIP_VERSION=21.1.2
ARG PYTHON_VERSION=3.8.10
ARG JUPYTERLAB_VERSION=4.0.7
ARG JUPYTER_VERSION=7.0.5
ARG MINIFORGE_VERSION=23.3.1-1
ARG NODE_MAJOR_VERSION=18
ARG PIP_VERSION=23.2.1
ARG PYTHON_VERSION=3.11.6
# install -- node.js
RUN export DEBIAN_FRONTEND=noninteractive \
&& curl -sL "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" | apt-key add - \
&& echo "deb https://deb.nodesource.com/node_14.x focal main" > /etc/apt/sources.list.d/nodesource.list \
&& curl -sL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | apt-key add - \
&& echo "deb https://deb.nodesource.com/node_${NODE_MAJOR_VERSION}.x nodistro main" > /etc/apt/sources.list.d/nodesource.list \
&& apt-get -yq update \
&& apt-get -yq install --no-install-recommends \
nodejs \
@ -33,12 +39,18 @@ RUN mkdir -p ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
# switch to NB_UID for installs
USER ${NB_UID}
USER $NB_UID
# install - conda, pip, python
RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh" -o /tmp/Miniforge3.sh \
&& curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh.sha256" -o /tmp/Miniforge3.sh.sha256 \
&& echo "$(cat /tmp/Miniforge3.sh.sha256 | awk '{ print $1; }') /tmp/Miniforge3.sh" | sha256sum --check \
# install - conda, pip, python, jupyter
RUN case "${TARGETARCH}" in \
amd64) MINIFORGE_ARCH="x86_64" ;; \
arm64) MINIFORGE_ARCH="aarch64" ;; \
ppc64le) MINIFORGE_ARCH="ppc64le" ;; \
*) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \
esac \
&& curl -fsSL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh" -o /tmp/Miniforge3.sh \
&& curl -fsSL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh.sha256" -o /tmp/Miniforge3.sh.sha256 \
&& echo "$(cat /tmp/Miniforge3.sh.sha256 | awk '{ print $1; }') /tmp/Miniforge3.sh" | sha256sum -c - \
&& rm /tmp/Miniforge3.sh.sha256 \
&& /bin/bash /tmp/Miniforge3.sh -b -f -p ${CONDA_DIR} \
&& rm /tmp/Miniforge3.sh \
@ -51,29 +63,34 @@ RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIF
conda=${MINIFORGE_VERSION:0:-2} \
pip=${PIP_VERSION} \
&& conda update -y -q --all \
&& conda clean -a -f -y \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
&& conda clean -a -f -y
# install - jupyter
RUN echo "jupyterlab >=${JUPYTERLAB_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "notebook >=${JUPYTER_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& conda install -y -q \
jupyterlab==${JUPYTERLAB_VERSION} \
notebook==${JUPYTER_VERSION} \
&& conda clean -a -f -y
# install - requirements.txt
COPY --chown=jovyan:users requirements.txt /tmp
COPY --chown=${NB_USER}:users requirements.txt /tmp
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt \
&& jupyter lab --generate-config \
&& rm -rf ${HOME}/.cache/yarn \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
&& rm -f /tmp/requirements.txt
# s6 - copy scripts
COPY --chown=jovyan:users s6/ /etc
COPY --chown=${NB_USER}:users --chmod=755 s6/ /etc
# s6 - 01-copy-tmp-home
USER root
RUN mkdir -p /tmp_home \
&& cp -r ${HOME} /tmp_home \
&& chown -R ${NB_USER}:users /tmp_home
USER ${NB_UID}
USER $NB_UID
EXPOSE 8888
# generate jupyter config
RUN jupyter notebook --generate-config \
&& jupyter lab --generate-config \
&& jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
ENTRYPOINT ["/init"]
EXPOSE 8888

40
components/example-notebook-servers/jupyter/Makefile
View File

@ -1,11 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-base:
$(MAKE) docker-build -C ../base TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build: docker-build-base
docker build -t ${REGISTRY}/jupyter:${TAG} --build-arg BASE_IMG=${REGISTRY}/base:${TAG} -f Dockerfile .
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-push:
docker push ${REGISTRY}/jupyter:${TAG}
IMAGE_NAME := jupyter
BASE_IMAGE := $(REGISTRY)/base:$(TAG)
BASE_IMAGE_FOLDERS := base
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

3
components/example-notebook-servers/jupyter/requirements.txt
View File

@ -1,3 +0,0 @@
jupyterlab==3.4.3
notebook==6.4.12
ipykernel==6.15.0

2
components/example-notebook-servers/jupyter/s6/cont-init.d/01-copy-tmp-home
View File

@ -1,2 +1,2 @@
#!/usr/bin/with-contenv bash
#!/command/with-contenv bash
cp -r -n /tmp_home/* /home/

2
components/example-notebook-servers/jupyter/s6/services.d/jupyterlab/run
View File

@ -1,4 +1,4 @@
#!/usr/bin/with-contenv bash
#!/command/with-contenv bash
cd "${HOME}"
exec /opt/conda/bin/jupyter lab \
--notebook-dir="${HOME}" \

18
components/example-notebook-servers/rstudio-tidyverse/Dockerfile
View File

@ -1,20 +1,14 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<rstudio>
FROM $BASE_IMG
# args - software versions
ARG R_TIDYVERSE_VERSION="1.3.1"
# switch to root user for conda installation
# (only necessary when installing r-tidyverse with Kaniko)
USER root
ARG R_TIDYVERSE_VERSION="2.0.0"
# install - r-tidyverse
RUN conda install -y -q \
r-tidyverse=${R_TIDYVERSE_VERSION} \
&& conda update -y -q --all \
&& conda clean -a -f -y \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
USER ${NB_USER}
&& conda clean -a -f -y

40
components/example-notebook-servers/rstudio-tidyverse/Makefile
View File

@ -1,11 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-rstudio:
$(MAKE) docker-build -C ../rstudio TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build: docker-build-rstudio
docker build -t ${REGISTRY}/rstudio-tidyverse:${TAG} --build-arg BASE_IMG=${REGISTRY}/rstudio:${TAG} -f Dockerfile .
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-push:
docker push ${REGISTRY}/rstudio-tidyverse:${TAG}
IMAGE_NAME := rstudio-tidyverse
BASE_IMAGE := $(REGISTRY)/rstudio:$(TAG)
BASE_IMAGE_FOLDERS := rstudio
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

84
components/example-notebook-servers/rstudio/Dockerfile
View File

@ -1,21 +1,24 @@
# Use the respective Makefile to pass the appropriate BASE_IMG and build the image correctly
#
# NOTE: Use the Makefiles to build this image correctly.
#
ARG BASE_IMG=<base>
FROM $BASE_IMG
USER root
ARG TARGETARCH
# args - software versions
ARG MINIFORGE_ARCH="x86_64"
# renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
ARG MINIFORGE_VERSION=4.10.1-4
ARG PIP_VERSION=21.1.2
ARG PYTHON_VERSION=3.8.10
ARG RSTUDIO_ARCH="amd64"
# renovate: datasource=github-tags depName=rstudio/rstudio versioning=semver
ARG RSTUDIO_VERSION=2022.12.0+353
ARG R_BASE_VERSION=4.1.0
ARG R_RETICULATE_VERSION=1.20
ARG R_PNG_VERSION=0.1_7
# args - software versions (python)
ARG MINIFORGE_VERSION=23.3.1-1
ARG PIP_VERSION=23.2.1
ARG PYTHON_VERSION=3.11.6
# args - software versions (R)
ARG RSTUDIO_VERSION=2023.09.0-463
ARG R_BASE_VERSION=4.3.1
ARG R_RETICULATE_VERSION=1.34.0
ARG R_PNG_VERSION=0.1_8
USER root
# install - binary rstudio dependencies
RUN apt-get -yq update \
@ -26,6 +29,7 @@ RUN apt-get -yq update \
libclang-dev \
libedit2 \
libpq5 \
libssl-dev \
psmisc \
rrdtool \
sudo \
@ -46,12 +50,18 @@ RUN mkdir -p ${CONDA_DIR} \
# setup environment for R
ENV R_HOME ${CONDA_DIR}/lib/R
USER ${NB_UID}
USER $NB_UID
# install - conda, pip, python, r-base
RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh" -o /tmp/Miniforge3.sh \
&& curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh.sha256" -o /tmp/Miniforge3.sh.sha256 \
&& echo "$(cat /tmp/Miniforge3.sh.sha256 | awk '{ print $1; }') /tmp/Miniforge3.sh" | sha256sum --check \
# install - conda, pip, python
RUN case "${TARGETARCH}" in \
amd64) MINIFORGE_ARCH="x86_64" ;; \
arm64) MINIFORGE_ARCH="aarch64" ;; \
ppc64le) MINIFORGE_ARCH="ppc64le" ;; \
*) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \
esac \
&& curl -fsSL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh" -o /tmp/Miniforge3.sh \
&& curl -fsSL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh.sha256" -o /tmp/Miniforge3.sh.sha256 \
&& echo "$(cat /tmp/Miniforge3.sh.sha256 | awk '{ print $1; }') /tmp/Miniforge3.sh" | sha256sum -c - \
&& rm /tmp/Miniforge3.sh.sha256 \
&& /bin/bash /tmp/Miniforge3.sh -b -f -p ${CONDA_DIR} \
&& rm /tmp/Miniforge3.sh \
@ -59,21 +69,23 @@ RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIF
&& conda config --system --set show_channel_urls true \
&& echo "conda ${MINIFORGE_VERSION:0:-2}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "python ${PYTHON_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "r-base ${R_BASE_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& conda install -y -q \
python=${PYTHON_VERSION} \
conda=${MINIFORGE_VERSION:0:-2} \
pip=${PIP_VERSION} \
&& conda update -y -q --all \
&& conda clean -a -f -y
# install - r packages
RUN echo "r-base ${R_BASE_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& conda install -y -q \
r-base=${R_BASE_VERSION} \
r-reticulate=${R_RETICULATE_VERSION} \
r-png=${R_PNG_VERSION} \
&& conda update -y -q --all \
&& conda clean -a -f -y \
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
&& chown -R ${NB_USER}:users ${HOME}
&& conda clean -a -f -y
# set default CRAN repo to RSPM (it has pre-compiled R packages, increasing user install speed)
RUN echo 'options(repos=c(CRAN="https://packagemanager.rstudio.com/all/__linux__/focal/latest"))' >> ${R_HOME}/etc/Rprofile.site \
RUN echo 'options(repos=c(CRAN="https://packagemanager.rstudio.com/all/__linux__/jammy/latest"))' >> ${R_HOME}/etc/Rprofile.site \
&& echo 'options(HTTPUserAgent=sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version$platform, R.version$arch, R.version$os)))' >> ${R_HOME}/etc/Rprofile.site
# R needs TZ set
@ -83,17 +95,19 @@ RUN echo "TZ=${TZ}" >> ${R_HOME}/etc/Renviron.site
USER root
# install - rstudio-server
# Affero General Public License may apply to RStudio: https://www.gnu.org/licenses/agpl-3.0.en.html
RUN curl -sL "https://download2.rstudio.org/server/bionic/${RSTUDIO_ARCH}/rstudio-server-${RSTUDIO_VERSION//+/-}-${RSTUDIO_ARCH}.deb" -o /tmp/rstudio-server.deb \
# add rstudio public code-signing key
RUN case "${TARGETARCH}" in \
amd64) RSTUDIO_DEB_URL="https://download2.rstudio.org/server/jammy/amd64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb" ;; \
arm64) RSTUDIO_DEB_URL="https://s3.amazonaws.com/rstudio-ide-build/server/jammy/arm64/rstudio-server-${RSTUDIO_VERSION}-arm64.deb" ;; \
*) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \
esac \
&& curl -fsSL "${RSTUDIO_DEB_URL}" -o /tmp/rstudio-server.deb \
# add rstudio public code-signing keys
&& gpg --keyserver keys.gnupg.net --keyserver pgp.surfnet.nl --recv-keys 3F32EE77E331692F \
&& gpg --keyserver keys.openpgp.org --recv-keys 51C0B5BB19F92D60 \
# validate the build signature
&& dpkg-sig --verify /tmp/rstudio-server.deb \
&& dpkg -i /tmp/rstudio-server.deb \
&& rm -f /tmp/rstudio-server.deb \
# our Kaniko build excludes `/var/run/*` but RStudio needs those files,
# so we move them to `/run`, wich is actually the same place due to symbolic links
&& mv -n /var/run/rstudio-server* /run \
# use advisory file-locks to improve PVC support
&& echo "lock-type=advisory" > /etc/rstudio/file-locks \
# allow kubeflow to display rstudio in an iframe
@ -105,19 +119,19 @@ RUN curl -sL "https://download2.rstudio.org/server/bionic/${RSTUDIO_ARCH}/rstudi
&& chown -R ${NB_USER}:users /var/lib/rstudio-server
# tell rstudio to use conda python by setting `RETICULATE_PYTHON` with `--rsession-path=/opt/rsession.sh`
COPY --chown=jovyan:users rsession.sh /opt
COPY --chown=${NB_USER}:users --chmod=755 rsession.sh /opt
RUN chmod +x /opt/rsession.sh
# s6 - copy scripts
COPY --chown=jovyan:users s6/ /etc
COPY --chown=${NB_USER}:users --chmod=755 s6/ /etc
# s6 - 01-copy-tmp-home
RUN mkdir -p /tmp_home \
&& cp -r ${HOME} /tmp_home \
&& chown -R ${NB_USER}:users /tmp_home
USER ${NB_UID}
USER $NB_UID
EXPOSE 8888
ENTRYPOINT ["/init"]
ENTRYPOINT ["/init"]

40
components/example-notebook-servers/rstudio/Makefile
View File

@ -1,11 +1,35 @@
TAG ?= $(shell git describe --tags --always --dirty)
REGISTRY ?= kubeflownotebookswg
#
# This Makefile is templated, the following targets are provided:
# - Current Architecture:
# - docker-build: build the docker image
# - docker-build-dep: build the docker image (and any base images)
# - docker-push: push the docker image
# - docker-push-dep: push the docker image (and any base images)
# - Multi-Architecture:
# - docker-build-multi-arch: build the docker image
# - docker-build-multi-arch-dep: build the docker image (and any base images)
# - docker-build-push-multi-arch: build AND push the docker image
# - docker-build-push-multi-arch-dep: build AND push the docker image (and any base images)
#
docker-build-base:
$(MAKE) docker-build -C ../base TAG=${TAG}
GIT_COMMIT := $(shell git rev-parse HEAD)
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
docker-build: docker-build-base
docker build -t ${REGISTRY}/rstudio:${TAG} --build-arg BASE_IMG=${REGISTRY}/base:${TAG} -f Dockerfile .
REGISTRY ?= docker.io/kubeflownotebookswg
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
docker-push:
docker push ${REGISTRY}/rstudio:${TAG}
IMAGE_NAME := rstudio
BASE_IMAGE := $(REGISTRY)/base:$(TAG)
BASE_IMAGE_FOLDERS := base
# we use a separate registry-type cache to keep the main registry clean
# https://docs.docker.com/build/cache/backends/
CACHE_IMAGE ?= ghcr.io/kubeflow/kubeflow/notebook-servers/build-cache
CACHE_TAG ?= $(IMAGE_NAME)
# https://docs.docker.com/engine/reference/commandline/buildx_build/#platform
ARCH ?= linux/amd64
# include build targets from common
include ../common.mk

2
components/example-notebook-servers/rstudio/s6/cont-init.d/01-copy-tmp-home
View File

@ -1,2 +1,2 @@
#!/usr/bin/with-contenv bash
#!/command/with-contenv bash
cp -r -n /tmp_home/* /home/

2
components/example-notebook-servers/rstudio/s6/cont-init.d/02-rstudio-env-fix
View File

@ -1,4 +1,4 @@
#!/usr/bin/with-contenv bash
#!/command/with-contenv bash
# rstudio terminal cant see environment variables set by the container runtime
# (which breaks kubectl, to fix this we store the KUBERNETES_* env vars in Renviron.site)
env | grep KUBERNETES_ >> ${R_HOME}/etc/Renviron.site

2
components/example-notebook-servers/rstudio/s6/services.d/rstudio/finish
View File

@ -1,2 +1,2 @@
#!/bin/bash
#!/command/with-contenv bash
exec rstudio-server stop

10
components/example-notebook-servers/rstudio/s6/services.d/rstudio/run
View File

@ -1,13 +1,13 @@
#!/usr/bin/with-contenv bash
#!/command/with-contenv bash
# using rstudio with non-root and `--auth-none=1` inexplicably requires USER to be set
export USER=${NB_USER}
export USER="${NB_USER}"
exec /usr/lib/rstudio-server/bin/rserver \
--server-daemonize=0 \
--server-working-dir=${HOME} \
--server-user=${NB_USER} \
--server-working-dir="${HOME}" \
--server-user="${NB_USER}" \
--www-address=0.0.0.0 \
--www-port=8888 \
--auth-none=1 \
--rsession-which-r=$(which R) \
--rsession-which-r="$(which R)" \
--rsession-path=/opt/rsession.sh