diff --git a/components/example-notebook-servers/base/Dockerfile b/components/example-notebook-servers/base/Dockerfile index 06f60fc8..e906276e 100644 --- a/components/example-notebook-servers/base/Dockerfile +++ b/components/example-notebook-servers/base/Dockerfile @@ -39,8 +39,8 @@ ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2 # args - software versions # https://kubernetes.io/releases/ # https://github.com/just-containers/s6-overlay/releases -ARG KUBECTL_VERSION=v1.29.9 -ARG S6_VERSION=v3.2.0.0 +ARG KUBECTL_VERSION=v1.31.6 +ARG S6_VERSION=v3.2.0.2 # set shell to bash SHELL ["/bin/bash", "-c"] diff --git a/components/example-notebook-servers/codeserver-python/Dockerfile b/components/example-notebook-servers/codeserver-python/Dockerfile index 8451aab7..fd54a19e 100644 --- a/components/example-notebook-servers/codeserver-python/Dockerfile +++ b/components/example-notebook-servers/codeserver-python/Dockerfile @@ -10,11 +10,15 @@ ARG TARGETARCH USER root # args - software versions -# https://github.com/microsoft/vscode-python/releases -ARG CODESERVER_PYTHON_VERSION=2024.14.1 -ARG MINIFORGE_VERSION=24.7.1-2 -ARG PIP_VERSION=24.2 -ARG PYTHON_VERSION=3.11.10 +# https://open-vsx.org/extension/ms-python/python +# https://open-vsx.org/extension/ms-toolsai/jupyter +# https://github.com/ipython/ipykernel/releases +ARG CODESERVER_PYTHON_VERSION=2025.0.0 +ARG CODESERVER_JUPYTER_VERSION=2024.11.0 +ARG IPYKERNEL_VERSION=6.29.5 +ARG MINIFORGE_VERSION=24.11.3-0 +ARG PIP_VERSION=24.3.1 +ARG PYTHON_VERSION=3.11.11 # setup environment for conda ENV CONDA_DIR /opt/conda @@ -52,6 +56,13 @@ RUN case "${TARGETARCH}" in \ && conda update -y -q --all \ && conda clean -a -f -y +# install - ipykernel +# NOTE: we need this for jupyter codeserver extension to work +RUN echo "ipykernel ==${IPYKERNEL_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \ + && conda install -y -q \ + ipykernel==${IPYKERNEL_VERSION} \ + && conda clean -a -f -y + # install - requirements.txt COPY --chown=${NB_USER}:${NB_GID} requirements.txt /tmp RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \ @@ -59,8 +70,12 @@ RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \ # install - codeserver extensions RUN code-server --install-extension "ms-python.python@${CODESERVER_PYTHON_VERSION}" --force \ + && code-server --install-extension "ms-toolsai.jupyter@${CODESERVER_JUPYTER_VERSION}" --force \ && code-server --list-extensions --show-versions +# home - pre-populate home with files for this image +COPY --chown=${NB_USER}:${NB_GID} home/. ${HOME}/ + # s6 - 01-copy-tmp-home # NOTE: the contents of $HOME_TMP are copied to $HOME at runtime # this is a workaround because a PVC will be mounted at $HOME diff --git a/components/example-notebook-servers/codeserver-python/home/README__CODESERVER_PYTHON.md b/components/example-notebook-servers/codeserver-python/home/README__CODESERVER_PYTHON.md new file mode 100644 index 00000000..4127c43b --- /dev/null +++ b/components/example-notebook-servers/codeserver-python/home/README__CODESERVER_PYTHON.md @@ -0,0 +1,13 @@ +# About the Code-Server Python Image + +This file contains notes about the Kubeflow Notebooks _Code-Server Python_ image. + +## Jupyter Extension required HTTPS + +This image comes with the jupyter extension installed, which allows you to run and edit jupyter notebooks in code-server. + +However, because the jupyter extension uses [Service Workers](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API), it requires HTTPS to work. +That is, if you access this notebook over HTTP, the Jupyter extension will NOT work. + +Additionally, if you are using __Chrome__, the HTTPS certificate must be __valid__ and trusted by your browser. +As a workaround, if you have a self-signed HTTPS certificate, you could use Firefox, or set the [`unsafely-treat-insecure-origin-as-secure`](chrome://flags/#unsafely-treat-insecure-origin-as-secure) flag in Chrome. \ No newline at end of file diff --git a/components/example-notebook-servers/codeserver-python/requirements.txt b/components/example-notebook-servers/codeserver-python/requirements.txt index 86b05082..a624253f 100644 --- a/components/example-notebook-servers/codeserver-python/requirements.txt +++ b/components/example-notebook-servers/codeserver-python/requirements.txt @@ -1,2 +1,2 @@ # kubeflow packages -kfp==2.9.0 \ No newline at end of file +kfp==2.11.0 \ No newline at end of file diff --git a/components/example-notebook-servers/codeserver/Dockerfile b/components/example-notebook-servers/codeserver/Dockerfile index fed0f9ad..81a62399 100644 --- a/components/example-notebook-servers/codeserver/Dockerfile +++ b/components/example-notebook-servers/codeserver/Dockerfile @@ -9,7 +9,7 @@ ARG TARGETARCH # args - software versions # https://github.com/coder/code-server/releases -ARG CODESERVER_VERSION=4.93.1 +ARG CODESERVER_VERSION=4.96.4 USER root diff --git a/components/example-notebook-servers/codeserver/s6/services.d/code-server/run b/components/example-notebook-servers/codeserver/s6/services.d/code-server/run index 74e28816..828dc599 100644 --- a/components/example-notebook-servers/codeserver/s6/services.d/code-server/run +++ b/components/example-notebook-servers/codeserver/s6/services.d/code-server/run @@ -1,6 +1,13 @@ #!/command/with-contenv bash -cd "${HOME}" +# many tools default to using the home directory for runtime data. +# as we mount a PVC to the home directory, these files might have the wrong permissions. +# to avoid this, we set the runtime directory to a path that is NOT a persistent volume. +export XDG_RUNTIME_DIR="/tmp/runtime-${NB_USER}" +mkdir -p "${XDG_RUNTIME_DIR}" +chmod 700 "${XDG_RUNTIME_DIR}" + +cd "${HOME}" || { echo "ERROR: failed to cd to ${HOME}"; exit 1; } echo "INFO: starting code-server..." exec 2>&1 exec /usr/bin/code-server \ diff --git a/components/example-notebook-servers/jupyter-pytorch-cuda-full/Dockerfile b/components/example-notebook-servers/jupyter-pytorch-cuda-full/Dockerfile index 2e9afc81..b6b18095 100644 --- a/components/example-notebook-servers/jupyter-pytorch-cuda-full/Dockerfile +++ b/components/example-notebook-servers/jupyter-pytorch-cuda-full/Dockerfile @@ -8,18 +8,18 @@ FROM $BASE_IMG # install - conda packages # NOTE: we use mamba to speed things up RUN mamba install -y -q \ - bokeh==3.3.4 \ - cloudpickle==2.2.1 \ - dill==0.3.8 \ - ipympl==0.9.4 \ - matplotlib==3.8.4 \ - numpy==1.24.4 \ - pandas==2.1.4 \ - scikit-image==0.22.0 \ - scikit-learn==1.3.2 \ - scipy==1.11.3 \ + bokeh==3.6.3 \ + cloudpickle==3.1.1 \ + dill==0.3.9 \ + ipympl==0.9.6 \ + matplotlib==3.10.0 \ + numpy==1.26.4 \ + pandas==2.2.3 \ + scikit-image==0.25.1 \ + scikit-learn==1.6.1 \ + scipy==1.15.1 \ seaborn==0.13.2 \ - xgboost==1.7.6 \ + xgboost==2.1.4 \ && mamba clean -a -f -y # install - requirements.txt diff --git a/components/example-notebook-servers/jupyter-pytorch-cuda-full/requirements.txt b/components/example-notebook-servers/jupyter-pytorch-cuda-full/requirements.txt index 5c7e8607..2bd44e55 100644 --- a/components/example-notebook-servers/jupyter-pytorch-cuda-full/requirements.txt +++ b/components/example-notebook-servers/jupyter-pytorch-cuda-full/requirements.txt @@ -1,5 +1,5 @@ # kubeflow packages -kfp==2.9.0 +kfp==2.11.0 # jupyterlab extensions -jupyterlab-git==0.50.1 \ No newline at end of file +jupyterlab-git==0.50.2 \ No newline at end of file diff --git a/components/example-notebook-servers/jupyter-pytorch-cuda/Dockerfile b/components/example-notebook-servers/jupyter-pytorch-cuda/Dockerfile index c3a7f196..e49df09a 100644 --- a/components/example-notebook-servers/jupyter-pytorch-cuda/Dockerfile +++ b/components/example-notebook-servers/jupyter-pytorch-cuda/Dockerfile @@ -9,18 +9,23 @@ FROM $BASE_IMG # https://github.com/pytorch/pytorch/releases # https://github.com/pytorch/audio/releases # https://github.com/pytorch/vision/releases -ARG PYTORCH_VERSION=2.3.1 -ARG TORCHAUDIO_VERSION=2.3.1 -ARG TORCHVISION_VERSION=0.18.1 +# NOTE: correct version alignments are found at https://pytorch.org/get-started/previous-versions/ +# they sometimes forget to publish a github release of `torchaudio` or `torchvision` when the cut a +# patch release of `torch`, but they still publish a patch of `torchaudio` and `torchvision` on pypi +# TODO: it seems like torch 2.6.0+ with CUDA 12.6+ will have experimental aarch64 support +# when updating past that version, remember to test building these images on arm +ARG PYTORCH_VERSION=2.5.1 +ARG TORCHAUDIO_VERSION=2.5.1 +ARG TORCHVISION_VERSION=0.20.1 # nvidia container toolkit # https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -ENV NVIDIA_REQUIRE_CUDA "cuda>=12.1" +ENV NVIDIA_REQUIRE_CUDA "cuda>=12.4" # install - pytorch (cuda) -RUN python3 -m pip install --quiet --no-cache-dir --index-url https://download.pytorch.org/whl/cu121 \ +RUN python3 -m pip install --quiet --no-cache-dir --index-url https://download.pytorch.org/whl/cu124 \ torch==${PYTORCH_VERSION} \ torchaudio==${TORCHAUDIO_VERSION} \ torchvision==${TORCHVISION_VERSION} diff --git a/components/example-notebook-servers/jupyter-pytorch-full/Dockerfile b/components/example-notebook-servers/jupyter-pytorch-full/Dockerfile index f393ef17..50d227c0 100644 --- a/components/example-notebook-servers/jupyter-pytorch-full/Dockerfile +++ b/components/example-notebook-servers/jupyter-pytorch-full/Dockerfile @@ -8,18 +8,18 @@ FROM $BASE_IMG # install - conda packages # NOTE: we use mamba to speed things up RUN mamba install -y -q \ - bokeh==3.3.4 \ - cloudpickle==2.2.1 \ - dill==0.3.8 \ - ipympl==0.9.4 \ - matplotlib==3.8.4 \ - numpy==1.24.4 \ - pandas==2.1.4 \ - scikit-image==0.22.0 \ - scikit-learn==1.3.2 \ - scipy==1.11.3 \ + bokeh==3.6.3 \ + cloudpickle==3.1.1 \ + dill==0.3.9 \ + ipympl==0.9.6 \ + matplotlib==3.10.0 \ + numpy==1.26.4 \ + pandas==2.2.3 \ + scikit-image==0.25.1 \ + scikit-learn==1.6.1 \ + scipy==1.15.1 \ seaborn==0.13.2 \ - xgboost==1.7.6 \ + xgboost==2.1.4 \ && mamba clean -a -f -y # install - requirements.txt diff --git a/components/example-notebook-servers/jupyter-pytorch-full/requirements.txt b/components/example-notebook-servers/jupyter-pytorch-full/requirements.txt index 5c7e8607..2bd44e55 100644 --- a/components/example-notebook-servers/jupyter-pytorch-full/requirements.txt +++ b/components/example-notebook-servers/jupyter-pytorch-full/requirements.txt @@ -1,5 +1,5 @@ # kubeflow packages -kfp==2.9.0 +kfp==2.11.0 # jupyterlab extensions -jupyterlab-git==0.50.1 \ No newline at end of file +jupyterlab-git==0.50.2 \ No newline at end of file diff --git a/components/example-notebook-servers/jupyter-pytorch-gaudi-full/Dockerfile b/components/example-notebook-servers/jupyter-pytorch-gaudi-full/Dockerfile index 9efdbcd4..bcff4311 100644 --- a/components/example-notebook-servers/jupyter-pytorch-gaudi-full/Dockerfile +++ b/components/example-notebook-servers/jupyter-pytorch-gaudi-full/Dockerfile @@ -8,18 +8,18 @@ FROM $BASE_IMG # install - conda packages # NOTE: we use mamba to speed things up RUN mamba install -y -q \ - bokeh==3.3.4 \ - cloudpickle==2.2.1 \ - dill==0.3.8 \ - ipympl==0.9.4 \ - matplotlib==3.8.4 \ - numpy==1.24.4 \ - pandas==2.1.4 \ - scikit-image==0.22.0 \ - scikit-learn==1.3.2 \ - scipy==1.11.3 \ + bokeh==3.6.3 \ + cloudpickle==3.1.1 \ + dill==0.3.9 \ + ipympl==0.9.6 \ + matplotlib==3.10.0 \ + numpy==1.26.4 \ + pandas==2.2.3 \ + scikit-image==0.25.1 \ + scikit-learn==1.6.1 \ + scipy==1.15.1 \ seaborn==0.13.2 \ - xgboost==1.7.6 \ + xgboost==2.1.4 \ && mamba clean -a -f -y # install - requirements.txt diff --git a/components/example-notebook-servers/jupyter-pytorch-gaudi-full/requirements.txt b/components/example-notebook-servers/jupyter-pytorch-gaudi-full/requirements.txt index b8023da1..d030f7cf 100644 --- a/components/example-notebook-servers/jupyter-pytorch-gaudi-full/requirements.txt +++ b/components/example-notebook-servers/jupyter-pytorch-gaudi-full/requirements.txt @@ -1,5 +1,5 @@ # kubeflow packages -kfp==2.9.0 +kfp==2.11.0 # jupyterlab extensions -jupyterlab-git==0.50.1 +jupyterlab-git==0.50.2 diff --git a/components/example-notebook-servers/jupyter-pytorch-gaudi/Dockerfile b/components/example-notebook-servers/jupyter-pytorch-gaudi/Dockerfile index 73cc8163..a3eac2d2 100644 --- a/components/example-notebook-servers/jupyter-pytorch-gaudi/Dockerfile +++ b/components/example-notebook-servers/jupyter-pytorch-gaudi/Dockerfile @@ -164,8 +164,8 @@ COPY --chown=${NB_USER}:users requirements.txt /tmp RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \ && rm -f /tmp/requirements.txt -# install - note about Gaudi workloads and hugepages -COPY README.md "${HOME}" +# home - pre-populate home with files for this image +COPY --chown=${NB_USER}:${NB_GID} home/. ${HOME}/ # s6 - 01-copy-tmp-home # NOTE: the contents of $HOME_TMP are copied to $HOME at runtime diff --git a/components/example-notebook-servers/jupyter-pytorch-gaudi/README.md b/components/example-notebook-servers/jupyter-pytorch-gaudi/README.md deleted file mode 100644 index 8a37e3c1..00000000 --- a/components/example-notebook-servers/jupyter-pytorch-gaudi/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# A note about Gaudi workloads - -Some Gaudi workloads require hugepages to run correctly. One can find a list of workloads listed [here](https://console.cloud.intel.com/docs/guides/k8s_guide.html#hugepages-settings-by-model). As of now, Kubeflow Notebooks lack the capability to request hugepages resources for individual notebooks. - -To add hugepages resources for the Notebook Pod, there are a couple of options: -* Use [Limit Ranges](https://kubernetes.io/docs/concepts/policy/limit-range/) to set default hugepages amount for all Pods in the namespace. -* Edit the Kubeflow's notebook object after it has been created and add the hugepages resources to the pod template. - -The upcoming Notebooks 2.0 release will improve the situation with its `podTemplate` feature. diff --git a/components/example-notebook-servers/jupyter-pytorch-gaudi/home/README__PYTORCH_GAUDI.md b/components/example-notebook-servers/jupyter-pytorch-gaudi/home/README__PYTORCH_GAUDI.md new file mode 100644 index 00000000..dc9c8870 --- /dev/null +++ b/components/example-notebook-servers/jupyter-pytorch-gaudi/home/README__PYTORCH_GAUDI.md @@ -0,0 +1,15 @@ +# About the PyTorch Gaudi Image + +This file contains notes about the Kubeflow Notebooks _PyTorch Gaudi_ image. + +## Hugepages Support + +Some Gaudi workloads require hugepages to run correctly. +One can find a list of workloads listed [here](https://console.cloud.intel.com/docs/guides/k8s_guide.html#hugepages-settings-by-model). +As of now, Kubeflow Notebooks lacks the capability to request hugepages resources for individual notebooks. + +To add hugepages resources for the Notebook Pod, there are a couple of options: +* Use [Limit Ranges](https://kubernetes.io/docs/concepts/policy/limit-range/) to set a default hugepages amount for all Pods in the namespace. +* Edit the Notebook object after it has been created and add the hugepages resources to the pod template. + +The upcoming Notebooks 2.0 release will improve the situation with its `podTemplate` feature. diff --git a/components/example-notebook-servers/jupyter-pytorch/Dockerfile b/components/example-notebook-servers/jupyter-pytorch/Dockerfile index bb0ff1d7..2848ae15 100644 --- a/components/example-notebook-servers/jupyter-pytorch/Dockerfile +++ b/components/example-notebook-servers/jupyter-pytorch/Dockerfile @@ -9,9 +9,12 @@ FROM $BASE_IMG # https://github.com/pytorch/pytorch/releases # https://github.com/pytorch/audio/releases # https://github.com/pytorch/vision/releases -ARG PYTORCH_VERSION=2.3.1 -ARG TORCHAUDIO_VERSION=2.3.1 -ARG TORCHVISION_VERSION=0.18.1 +# NOTE: correct version alignments are found at https://pytorch.org/get-started/previous-versions/ +# they sometimes forget to publish a github release of `torchaudio` or `torchvision` when the cut a +# patch release of `torch`, but they still publish a patch of `torchaudio` and `torchvision` on pypi +ARG PYTORCH_VERSION=2.5.1 +ARG TORCHAUDIO_VERSION=2.5.1 +ARG TORCHVISION_VERSION=0.20.1 # install - pytorch (cpu) RUN python3 -m pip install --quiet --no-cache-dir --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \ diff --git a/components/example-notebook-servers/jupyter-scipy/Dockerfile b/components/example-notebook-servers/jupyter-scipy/Dockerfile index 5782b950..3ed04850 100644 --- a/components/example-notebook-servers/jupyter-scipy/Dockerfile +++ b/components/example-notebook-servers/jupyter-scipy/Dockerfile @@ -22,32 +22,34 @@ USER $NB_UID # NOTE: we use mamba to speed things up RUN mamba install -y -q \ altair \ - beautifulsoup4==4.12.3 \ - bokeh==3.3.4 \ + beautifulsoup4==4.13.3 \ + bokeh==3.6.3 \ bottleneck \ brotli \ cloudpickle \ - dask==2024.5.1 \ + dask==2024.12.1 \ dill \ h5py \ ipympl \ - matplotlib==3.8.4 \ + matplotlib==3.10.0 \ numba \ numexpr \ - openblas==0.3.25 \ + numpy==1.26.4 \ + openblas==0.3.28 \ openpyxl \ - pandas==2.1.4 \ + pandas==2.2.3 \ patsy \ protobuf \ - pytables==3.9.2 \ - scikit-image==0.22.0 \ - scikit-learn==1.3.2 \ - scipy==1.11.3 \ + pytables==3.10.2 \ + scikit-image==0.25.1 \ + scikit-learn==1.6.1 \ + scipy==1.15.1 \ seaborn==0.13.2 \ - sqlalchemy==2.0.30 \ + sqlalchemy==2.0.38 \ statsmodels \ sympy \ vincent \ + xgboost==2.1.4 \ xlrd \ && mamba clean -a -f -y diff --git a/components/example-notebook-servers/jupyter-scipy/requirements.txt b/components/example-notebook-servers/jupyter-scipy/requirements.txt index 5c7e8607..2bd44e55 100644 --- a/components/example-notebook-servers/jupyter-scipy/requirements.txt +++ b/components/example-notebook-servers/jupyter-scipy/requirements.txt @@ -1,5 +1,5 @@ # kubeflow packages -kfp==2.9.0 +kfp==2.11.0 # jupyterlab extensions -jupyterlab-git==0.50.1 \ No newline at end of file +jupyterlab-git==0.50.2 \ No newline at end of file diff --git a/components/example-notebook-servers/jupyter-tensorflow-cuda-full/Dockerfile b/components/example-notebook-servers/jupyter-tensorflow-cuda-full/Dockerfile index d6d1b62d..6fd4ce26 100644 --- a/components/example-notebook-servers/jupyter-tensorflow-cuda-full/Dockerfile +++ b/components/example-notebook-servers/jupyter-tensorflow-cuda-full/Dockerfile @@ -8,18 +8,18 @@ FROM $BASE_IMG # install - conda packages # NOTE: we use mamba to speed things up RUN mamba install -y -q \ - bokeh==3.3.4 \ - cloudpickle==2.2.1 \ - dill==0.3.8 \ - ipympl==0.9.4 \ - matplotlib==3.8.4 \ - numpy==1.24.4 \ - pandas==2.1.4 \ - scikit-image==0.22.0 \ - scikit-learn==1.3.2 \ - scipy==1.11.3 \ + bokeh==3.6.3 \ + cloudpickle==3.1.1 \ + dill==0.3.9 \ + ipympl==0.9.6 \ + matplotlib==3.10.0 \ + numpy==1.26.4 \ + pandas==2.2.3 \ + scikit-image==0.25.1 \ + scikit-learn==1.6.1 \ + scipy==1.15.1 \ seaborn==0.13.2 \ - xgboost==1.7.6 \ + xgboost==2.1.4 \ && mamba clean -a -f -y # install - requirements.txt diff --git a/components/example-notebook-servers/jupyter-tensorflow-cuda-full/requirements.txt b/components/example-notebook-servers/jupyter-tensorflow-cuda-full/requirements.txt index 5c7e8607..2bd44e55 100644 --- a/components/example-notebook-servers/jupyter-tensorflow-cuda-full/requirements.txt +++ b/components/example-notebook-servers/jupyter-tensorflow-cuda-full/requirements.txt @@ -1,5 +1,5 @@ # kubeflow packages -kfp==2.9.0 +kfp==2.11.0 # jupyterlab extensions -jupyterlab-git==0.50.1 \ No newline at end of file +jupyterlab-git==0.50.2 \ No newline at end of file diff --git a/components/example-notebook-servers/jupyter-tensorflow-cuda/Dockerfile b/components/example-notebook-servers/jupyter-tensorflow-cuda/Dockerfile index cac2c096..64a198a9 100644 --- a/components/example-notebook-servers/jupyter-tensorflow-cuda/Dockerfile +++ b/components/example-notebook-servers/jupyter-tensorflow-cuda/Dockerfile @@ -10,10 +10,10 @@ ARG TARGETARCH # args - software versions # - TensorFlow CUDA version matrix: https://www.tensorflow.org/install/source#gpu # - Extra PyPi from NVIDIA (for TensorRT): https://pypi.nvidia.com/ -# - TODO: it seems like TensorRT will be removed from TensorFlow in 2.18.0 -# when updating past that version, remember to remove all TensorRT -# related packages and configs in this Dockerfile -ARG TENSORFLOW_VERSION=2.15.1 +# - TODO: TensorRT will be removed from TensorFlow in 2.18.0 +# when updating past that version, either remove all TensorRT related packages, +# or investigate if there is a way to keep TensorRT support +ARG TENSORFLOW_VERSION=2.17.1 ARG TENSORRT_VERSION=8.6.1.post1 ARG TENSORRT_LIBS_VERSION=8.6.1 ARG TENSORRT_BINDINGS_VERSION=8.6.1 @@ -22,10 +22,12 @@ ARG TENSORRT_BINDINGS_VERSION=8.6.1 # https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -ENV NVIDIA_REQUIRE_CUDA "cuda>=12.2" +ENV NVIDIA_REQUIRE_CUDA "cuda>=12.3" # install - tensorflow -# - About '[and-cuda]' option: https://github.com/tensorflow/tensorflow/blob/v2.15.1/tensorflow/tools/pip_package/setup.py#L166-L177 +# - About '[and-cuda]' option: https://github.com/tensorflow/tensorflow/blob/v2.17.1/tensorflow/tools/pip_package/setup.py#L153-L164 +# - TODO: when updating TensorRT, you might need to change `tensorrt`, `tensorrt-libs`, and `tensorrt-bindings` +# to `tensorrt-cu12`, `tensorrt-cu12-libs`, and `tensorrt-cu12-bindings` respectively RUN python3 -m pip install --quiet --no-cache-dir --extra-index-url https://pypi.nvidia.com \ tensorflow[and-cuda]==${TENSORFLOW_VERSION} \ tensorrt==${TENSORRT_VERSION} \ @@ -34,8 +36,9 @@ RUN python3 -m pip install --quiet --no-cache-dir --extra-index-url https://pypi # create symlinks for TensorRT libs # - https://github.com/tensorflow/tensorflow/issues/61986#issuecomment-1880489731 -# - libnvinfer.so.8.6.1 -> libnvinfer.so.8 -# - libnvinfer_plugin.so.8.6.1 -> libnvinfer_plugin.so.8 +# - We are creating symlinks for the following libs, as this is where TF looks for them: +# - libnvinfer.so.8.6.1 -> libnvinfer.so.8 +# - libnvinfer_plugin.so.8.6.1 -> libnvinfer_plugin.so.8 ENV PYTHON_SITE_PACKAGES /opt/conda/lib/python3.11/site-packages ENV TENSORRT_LIBS ${PYTHON_SITE_PACKAGES}/tensorrt_libs RUN ln -s ${TENSORRT_LIBS}/libnvinfer.so.${TENSORRT_LIBS_VERSION%%.*} ${TENSORRT_LIBS}/libnvinfer.so.${TENSORRT_LIBS_VERSION} \ diff --git a/components/example-notebook-servers/jupyter-tensorflow-full/Dockerfile b/components/example-notebook-servers/jupyter-tensorflow-full/Dockerfile index f3901f44..3b7e3368 100644 --- a/components/example-notebook-servers/jupyter-tensorflow-full/Dockerfile +++ b/components/example-notebook-servers/jupyter-tensorflow-full/Dockerfile @@ -8,18 +8,18 @@ FROM $BASE_IMG # install - conda packages # NOTE: we use mamba to speed things up RUN mamba install -y -q \ - bokeh==3.3.4 \ - cloudpickle==2.2.1 \ - dill==0.3.8 \ - ipympl==0.9.4 \ - matplotlib==3.8.4 \ - numpy==1.24.4 \ - pandas==2.1.4 \ - scikit-image==0.22.0 \ - scikit-learn==1.3.2 \ - scipy==1.11.3 \ + bokeh==3.6.3 \ + cloudpickle==3.1.1 \ + dill==0.3.9 \ + ipympl==0.9.6 \ + matplotlib==3.10.0 \ + numpy==1.26.4 \ + pandas==2.2.3 \ + scikit-image==0.25.1 \ + scikit-learn==1.6.1 \ + scipy==1.15.1 \ seaborn==0.13.2 \ - xgboost==1.7.6 \ + xgboost==2.1.4 \ && mamba clean -a -f -y # install - requirements.txt diff --git a/components/example-notebook-servers/jupyter-tensorflow-full/requirements.txt b/components/example-notebook-servers/jupyter-tensorflow-full/requirements.txt index 5c7e8607..2bd44e55 100644 --- a/components/example-notebook-servers/jupyter-tensorflow-full/requirements.txt +++ b/components/example-notebook-servers/jupyter-tensorflow-full/requirements.txt @@ -1,5 +1,5 @@ # kubeflow packages -kfp==2.9.0 +kfp==2.11.0 # jupyterlab extensions -jupyterlab-git==0.50.1 \ No newline at end of file +jupyterlab-git==0.50.2 \ No newline at end of file diff --git a/components/example-notebook-servers/jupyter-tensorflow/Dockerfile b/components/example-notebook-servers/jupyter-tensorflow/Dockerfile index 63c7a9dc..b5f3ba2e 100644 --- a/components/example-notebook-servers/jupyter-tensorflow/Dockerfile +++ b/components/example-notebook-servers/jupyter-tensorflow/Dockerfile @@ -7,14 +7,7 @@ FROM $BASE_IMG # args - software versions # https://github.com/tensorflow/tensorflow/releases -ARG TENSORFLOW_VERSION=2.15.1 - -# install - h5py -# TODO: remove when upstream ARM wheels for `h5py` are available again: -# https://github.com/h5py/h5py/issues/2408 -RUN mamba install -y -q \ - h5py==3.11.0 \ - && mamba clean -a -f -y +ARG TENSORFLOW_VERSION=2.17.1 # install - tensorflow RUN python3 -m pip install --quiet --no-cache-dir \ diff --git a/components/example-notebook-servers/jupyter/Dockerfile b/components/example-notebook-servers/jupyter/Dockerfile index 51103258..7f4eeb13 100644 --- a/components/example-notebook-servers/jupyter/Dockerfile +++ b/components/example-notebook-servers/jupyter/Dockerfile @@ -12,12 +12,12 @@ USER root # args - software versions # https://github.com/jupyterlab/jupyterlab/releases # https://github.com/jupyter/notebook/releases -ARG JUPYTERLAB_VERSION=4.2.5 -ARG JUPYTER_VERSION=7.2.2 -ARG MINIFORGE_VERSION=24.7.1-2 +ARG JUPYTERLAB_VERSION=4.3.5 +ARG JUPYTER_VERSION=7.3.2 +ARG MINIFORGE_VERSION=24.11.3-0 ARG NODE_MAJOR_VERSION=20 -ARG PIP_VERSION=24.2 -ARG PYTHON_VERSION=3.11.10 +ARG PIP_VERSION=24.3.1 +ARG PYTHON_VERSION=3.11.11 # install -- node.js RUN export DEBIAN_FRONTEND=noninteractive \ diff --git a/components/example-notebook-servers/jupyter/s6/services.d/jupyterlab/run b/components/example-notebook-servers/jupyter/s6/services.d/jupyterlab/run index 8fa45b93..2f23715e 100644 --- a/components/example-notebook-servers/jupyter/s6/services.d/jupyterlab/run +++ b/components/example-notebook-servers/jupyter/s6/services.d/jupyterlab/run @@ -1,10 +1,18 @@ #!/command/with-contenv bash -# the runtime directory must be a path that is NOT a persistent volume +# many tools default to using the home directory for runtime data. +# as we mount a PVC to the home directory, these files might have the wrong permissions. +# to avoid this, we set the runtime directory to a path that is NOT a persistent volume. +export XDG_RUNTIME_DIR="/tmp/runtime-${NB_USER}" +mkdir -p "${XDG_RUNTIME_DIR}" +chmod 700 "${XDG_RUNTIME_DIR}" + +# the jupyter runtime directory must be a path that is NOT a persistent volume # as volumes often cause permission issues https://github.com/jupyter/notebook/issues/5058 +# note, jupyter ignores XDG_RUNTIME_DIR: https://github.com/jupyter/jupyter_core/pull/143 export JUPYTER_RUNTIME_DIR="/tmp/jupyter_runtime" -cd "${HOME}" +cd "${HOME}" || { echo "ERROR: failed to cd to ${HOME}"; exit 1; } echo "INFO: starting jupyter..." exec 2>&1 exec /opt/conda/bin/jupyter lab \ diff --git a/components/example-notebook-servers/rstudio/Dockerfile b/components/example-notebook-servers/rstudio/Dockerfile index 1a3a8280..8ff4e037 100644 --- a/components/example-notebook-servers/rstudio/Dockerfile +++ b/components/example-notebook-servers/rstudio/Dockerfile @@ -8,9 +8,9 @@ FROM $BASE_IMG ARG TARGETARCH # args - software versions (python) -ARG MINIFORGE_VERSION=24.7.1-2 -ARG PIP_VERSION=24.2 -ARG PYTHON_VERSION=3.11.10 +ARG MINIFORGE_VERSION=24.11.3-0 +ARG PIP_VERSION=24.3.1 +ARG PYTHON_VERSION=3.11.11 # args - software versions (R) # - rstudio: https://dailies.rstudio.com/release/ @@ -18,11 +18,11 @@ ARG PYTHON_VERSION=3.11.10 # - r-png: https://www.rforge.net/png/news.html # - r-reticulate: https://github.com/rstudio/reticulate/releases # - r-shiny: https://github.com/rstudio/shiny/releases -ARG RSTUDIO_VERSION=2024.09.0-375 -ARG R_BASE_VERSION=4.4.1 +ARG RSTUDIO_VERSION=2024.12.1-563 +ARG R_BASE_VERSION=4.4.2 ARG R_PNG_VERSION=0.1_8 -ARG R_RETICULATE_VERSION=1.39.0 -ARG R_SHINY_VERSION=1.9.1 +ARG R_RETICULATE_VERSION=1.40.0 +ARG R_SHINY_VERSION=1.10.0 USER root diff --git a/components/example-notebook-servers/rstudio/s6/services.d/rstudio/run b/components/example-notebook-servers/rstudio/s6/services.d/rstudio/run index a2a2e15e..ca4eb660 100644 --- a/components/example-notebook-servers/rstudio/s6/services.d/rstudio/run +++ b/components/example-notebook-servers/rstudio/s6/services.d/rstudio/run @@ -1,5 +1,12 @@ #!/command/with-contenv bash +# many tools default to using the home directory for runtime data. +# as we mount a PVC to the home directory, these files might have the wrong permissions. +# to avoid this, we set the runtime directory to a path that is NOT a persistent volume. +export XDG_RUNTIME_DIR="/tmp/runtime-${NB_USER}" +mkdir -p "${XDG_RUNTIME_DIR}" +chmod 700 "${XDG_RUNTIME_DIR}" + # the runtime state directory must be a path that is NOT a persistent volume # note, this env-var is not read by rstudio, but is used later in this script export RSTUDIO_RUNTIME_DIR="/tmp/rstudio_runtime"