chore: Remove pip requirements from docker files (#5064)

* Remove pip requirements from docker files

* Fix consistency and failing builds

* fixup arena requirements.txt

* Update KFserving to 0.4.1

* Remove pip requirements from docker files

* Fix consistency and failing builds

* fixup arena requirements.txt

* small fixup

backend/Dockerfile

@@ -12,7 +12,7 @@ FROM python:3.7 as compiler
 RUN apt-get update -y && apt-get install --no-install-recommends -y -q default-jdk python3-setuptools python3-dev jq
 RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
 COPY backend/requirements.txt .
-RUN python3 -m pip install -r requirements.txt
+RUN python3 -m pip install -r requirements.txt --no-cache-dir
 
 # Downloading Argo CLI so that the samples are validated
 #ADD  https://github.com/argoproj/argo/releases/download/v2.7.5/argo-linux-amd64 /usr/local/bin/argo

backend/Dockerfile.visualization

@@ -33,7 +33,7 @@ WORKDIR /src
 
 COPY backend/src/apiserver/visualization/requirements.txt /src
 
-RUN pip3 install -r requirements.txt
+RUN python3 -m pip install -r requirements.txt --no-cache-dir
 
 COPY backend/src/apiserver/visualization /src
 

components/arena/docker/requirements.txt

@@ -1,3 +1,3 @@
-requests
+requests==2.18.4
-six
+six==1.11.0
-pyyaml
+pyyaml==3.12

components/gcp/container/Dockerfile

@@ -18,8 +18,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 	wget patch \
 	&& rm -rf /var/lib/apt/lists/*
 
-RUN pip install apache-beam[gcp]
+COPY requirements.txt .
-RUN pip install pandas
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 ADD build /ml
 WORKDIR /ml

components/gcp/container/requirements.txt

@@ -0,0 +1,2 @@
+apache-beam[gcp]
+pandas

components/ibm-components/ffdl/serve/Dockerfile

@@ -1,6 +1,9 @@
 FROM python:3.6-slim
 
-RUN pip install kubernetes Flask flask-cors requests
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 ENV APP_HOME /app
 COPY src $APP_HOME

components/ibm-components/ffdl/serve/requirements.txt

@@ -0,0 +1,4 @@
+kubernetes
+Flask
+flask-cors
+requests

components/ibm-components/ffdl/train/Dockerfile

@@ -1,6 +1,9 @@
 FROM python:3.6-slim
 
-RUN pip install boto3 ruamel.yaml requests
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 ENV APP_HOME /app
 COPY src $APP_HOME

components/ibm-components/ffdl/train/requirements.txt

@@ -0,0 +1,3 @@
+boto3
+ruamel.yaml
+requests

components/ibm-components/watson/deploy/Dockerfile

@@ -5,7 +5,10 @@ RUN mkdir /app
 RUN mkdir /app/secrets
 
 # Watson studio and machine learning python client
-RUN pip install watson-machine-learning-client-V4>=1.0.110 minio
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 # Python functions with endpoints to Watson Machine Learning
 COPY src/wml-deploy.py /app

components/ibm-components/watson/deploy/requirements.txt

@@ -0,0 +1,2 @@
+watson-machine-learning-client-V4>=1.0.110
+minio

components/ibm-components/watson/store/Dockerfile

@@ -5,7 +5,10 @@ RUN mkdir /app
 RUN mkdir /app/secrets
 
 # Watson studio and machine learning python client
-RUN pip install watson-machine-learning-client-V4>=1.0.110 minio
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 # Python functions with endpoints to Watson Machine Learning
 COPY src/wml-store.py /app

components/ibm-components/watson/store/requirements.txt

@@ -0,0 +1,2 @@
+watson-machine-learning-client-V4>=1.0.110
+minio

components/ibm-components/watson/train/Dockerfile

@@ -5,7 +5,10 @@ RUN mkdir /app
 RUN mkdir /app/secrets
 
 # Watson studio and machine learning python client
-RUN pip install watson-machine-learning-client-V4>=1.0.110 minio
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 # Python functions with endpoints to Watson Machine Learning
 COPY src/wml-train.py /app

components/ibm-components/watson/train/requirements.txt

@@ -0,0 +1,2 @@
+watson-machine-learning-client-V4>=1.0.110
+minio

components/kubeflow/dnntrainer/Dockerfile

@@ -15,9 +15,10 @@
 ARG TF_TAG
 FROM tensorflow/tensorflow:$TF_TAG
 
-RUN pip install pyyaml==3.12 six==1.11.0 \
+COPY requirements.txt .
-    tensorflow-transform==0.23.0 \
+RUN python3 -m pip install -r \
-    tensorflow-model-analysis==0.23.0
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 ADD build /ml
 

components/kubeflow/dnntrainer/requirements.txt

@@ -0,0 +1,4 @@
+pyyaml==3.12
+six==1.11.0
+tensorflow-transform==0.23.0
+tensorflow-model-analysis==0.23.0

components/kubeflow/kfserving/Dockerfile

@@ -1,6 +1,9 @@
 FROM python:3.6-slim
 
-RUN pip3 install kubernetes==10.0.1 kfserving==0.4.1 requests==2.22.0 Flask==1.1.1 flask-cors==3.0.8
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 ENV APP_HOME /app
 COPY src $APP_HOME

components/kubeflow/kfserving/requirements.txt

@@ -0,0 +1,5 @@
+kubernetes==10.0.1
+kfserving==0.4.1
+requests==2.22.0
+Flask==1.1.1
+flask-cors==3.0.8

components/kubeflow/launcher/Dockerfile

@@ -14,7 +14,10 @@
 
 FROM python:3.6
 
-RUN pip install --no-cache-dir pyyaml kubernetes
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 ADD build /ml
 

components/kubeflow/launcher/requirements.txt

@@ -0,0 +1,2 @@
+pyyaml
+kubernetes

components/local/base/Dockerfile

@@ -14,9 +14,9 @@
 
 FROM python:3.7
 
-RUN pip install --no-cache-dir pandas==0.24.2
+COPY requirements.txt .
-RUN pip install --no-cache-dir scikit-learn==0.21.2
+RUN python3 -m pip install -r \
-RUN pip install --no-cache-dir scipy==1.4.1
+    requirements.txt --quiet --no-cache-dir \
-RUN pip install --no-cache-dir tensorflow==2.2.0
+    && rm -f requirements.txt
 
 ADD build /ml

components/local/base/requirements.txt

@@ -0,0 +1,4 @@
+pandas==0.24.2
+scikit-learn==0.21.2
+scipy==1.4.1
+tensorflow==2.2.0

components/presto/query/Dockerfile

@@ -12,6 +12,9 @@
 
 FROM python:3.7
 
-RUN python3 -m pip install pyhive[presto]
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 COPY ./src /pipelines/component/src

components/presto/query/requirements.txt

@@ -0,0 +1 @@
+pyhive[presto]

components/sample/keras/train_classifier/Dockerfile

@@ -1,5 +1,8 @@
 ARG BASE_IMAGE_TAG=1.12.0-py3
 FROM tensorflow/tensorflow:$BASE_IMAGE_TAG
-RUN python3 -m pip install keras
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 COPY ./src /pipelines/component/src
 ENTRYPOINT python3 /pipelines/component/src/train.py

components/sample/keras/train_classifier/requirements.txt

@@ -0,0 +1 @@
+keras

proxy/Dockerfile

@@ -16,7 +16,10 @@
 FROM gcr.io/inverting-proxy/agent@sha256:2e3c5ecd1a55b32056f3ce0c4aaac05e31b85c361d501bf3f8a81bec14c4fe87
 
 RUN apt-get update && apt-get install -y curl jq python-pip
-RUN pip install requests
+COPY requirements.txt .
+RUN python2 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 
 # Install gcloud SDK
 RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz

proxy/requirements.txt

@@ -0,0 +1 @@
+requests

samples/contrib/nvidia-resnet/components/preprocess/Dockerfile

@@ -14,7 +14,11 @@
 
 FROM nvcr.io/nvidia/tensorflow:19.03-py3
 
-RUN pip install keras
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
+
 ADD src /workspace
 WORKDIR /workspace
 

samples/contrib/nvidia-resnet/components/preprocess/requirements.txt

@@ -0,0 +1 @@
+keras

samples/contrib/nvidia-resnet/components/train/Dockerfile

@@ -14,7 +14,11 @@
 
 FROM nvcr.io/nvidia/tensorflow:19.03-py3
 
-RUN pip install keras
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
+
 ADD src /workspace
 WORKDIR /workspace
 

samples/contrib/nvidia-resnet/components/train/requirements.txt

@@ -0,0 +1 @@
+keras

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/download_dataset/Dockerfile

@@ -1,7 +1,9 @@
 FROM python:3.7
 ENV KAGGLE_USERNAME=[YOUR KAGGLE USERNAME] \
     KAGGLE_KEY=[YOUR KAGGLE KEY]
-RUN pip install kaggle
+COPY requirements.txt .
-RUN pip install google-cloud-storage
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 COPY ./download_data.py .
 CMD ["python", "download_data.py"]

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/download_dataset/requirements.txt

@@ -0,0 +1,2 @@
+kaggle
+google-cloud-storage

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/submit_result/Dockerfile

@@ -1,7 +1,9 @@
 FROM python:3.7
 ENV KAGGLE_USERNAME=[YOUR KAGGLE USERNAME] \
     KAGGLE_KEY=[YOUR KAGGLE KEY]
-RUN pip install kaggle
+COPY requirements.txt .
-RUN pip install gcsfs
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 COPY ./submit_result.py .
 CMD ["python", "submit_result.py"]

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/submit_result/requirements.txt

@@ -0,0 +1,2 @@
+kaggle
+gcsfs

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/train_model/Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.7
 COPY ./train.py .
-RUN pip install pandas==0.25.1
+COPY requirements.txt .
-RUN pip install gcsfs numpy matplotlib seaborn sklearn
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 CMD ["python", "train.py"]

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/train_model/requirements.txt

@@ -0,0 +1,6 @@
+pandas==0.25.1
+gcsfs
+numpy
+matplotlib
+seaborn
+sklearn

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/visualize_html/Dockerfile

@@ -1,4 +1,7 @@
 FROM tensorflow/tensorflow:2.0.0-py3
-RUN pip install gcsfs pandas matplotlib seaborn
+COPY requirements.txt .
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 COPY ./visualize.py .
 CMD ["python", 'visualize.py']

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/visualize_html/requirements.txt

@@ -0,0 +1,4 @@
+gcsfs
+pandas
+matplotlib
+seaborn

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/visualize_table/Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.7
-RUN pip install gcsfs
+COPY requirements.txt .
-RUN pip install pandas
+RUN python3 -m pip install -r \
+    requirements.txt --quiet --no-cache-dir \
+    && rm -f requirements.txt
 COPY ./visualize.py .
 CMD ["python", 'visualize.py']

samples/contrib/versioned-pipeline-ci-samples/kaggle-ci-sample/visualize_table/requirements.txt

@@ -0,0 +1,2 @@
+gcsfs
+pandas