chore(sdk): remove v1 dependencies from SDK v2 (#8668)

* split out deprecated requirements

* pip install deprecated requirements for sample tests

* remove superfluous comments in requirements.in

* update release notes

* ignore deprecated directory in unit tests

* remove unused argo compiler

* ignore deprecated path

* update run sample yaml

* install deprecated requirements for e2e tests

* add todos

backend/src/v2/test/components/run_sample.yaml

@@ -41,6 +41,8 @@ implementation:
       chmod +x "$backend_compiler_path"
       cp "$backend_compiler_path" /usr/local/bin/kfp-v2-compiler
       # run test sample
+      # TODO: remove deprecated dependency
+      pip install -r sdk/python/requirements-deprecated.txt
       python3 \
         -u \
         -m "$sample_path" \

backend/src/v2/test/sample-test.sh

@@ -23,6 +23,9 @@ source "${DIR}/scripts/ci-env.sh"
 
 # Install required packages from commit
 python3 -m pip install --upgrade pip
+
+# TODO: remove deprecated dependency
+python3 -m pip install -r $source_root/sdk/python/requirements-deprecated.txt
 python3 -m pip install $source_root/sdk/python
 
 # Install KFP server API from commit.

sdk/RELEASE.md

@@ -8,6 +8,7 @@
 
 ## Bug fixes and other changes
 * Accepts `PyYAML<7` in addition to `PyYAML>=5.3,<6` [\#8665](https://github.com/kubeflow/pipelines/pull/8665)
+* Remove v1 dependencies from SDK v2 [\#8668](https://github.com/kubeflow/pipelines/pull/8668)
 
 ## Documentation updates
 # 2.0.0-beta.10

sdk/python/requirements-deprecated.txt

@@ -0,0 +1,9 @@
+# TODO: remove deprecated dependency
+absl-py>=0.9,<2
+cloudpickle>=2.0.0,<3
+Deprecated>=1.2.7,<2
+fire>=0.3.1,<1
+jsonschema>=3.0.1,<4
+strip-hints>=0.1.8,<1
+uritemplate>=3.0.1,<4
+typer>=0.3.2,<1.0

sdk/python/requirements-dev.txt

@@ -1,3 +1,4 @@
+absl-py==1.4.0
 docformatter==1.4
 docker==5.0.3
 isort==5.10.1

sdk/python/requirements.in

@@ -2,26 +2,19 @@
 # the following in this folder:
 # pip-compile --no-emit-index-url requirements.in
 
-## kfp ##
-absl-py>=0.9,<2
 click>=7.1.2,<9
 docstring-parser>=0.7.3,<1
-# kfp.Client GCP auth
 # Pin google-api-core version for the bug fixing in 1.31.5
 # https://github.com/googleapis/python-api-core/releases/tag/v1.31.5
 google-api-core>=1.31.5,<3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0
 google-auth>=1.6.1,<3
 # https://github.com/googleapis/python-storage/blob/main/CHANGELOG.md#221-2022-03-15
 google-cloud-storage>=2.2.1,<3
-# NOTE: Maintainers, please do not require google-auth>=2.x.x
-# Until this issue is closed
-# https://github.com/googleapis/google-cloud-python/issues/10566
 kfp-pipeline-spec>=0.1.17,<0.2.0
 # Update the upper version whenever a new major version of the
 # kfp-server-api package is released.
 # Update the lower version when kfp sdk depends on new apis/fields in
 # kfp-server-api.
-# Note, please also update ./requirements.in
 kfp-server-api>=2.0.0a0,<3.0.0
 kubernetes>=8.0.0,<24
 protobuf>=3.13.0,<4
@@ -29,16 +22,5 @@ PyYAML>=5.3,<7
 requests-toolbelt>=0.8.0,<1
 tabulate>=0.8.6,<1
 
-## kfp.deprecated ##
-cloudpickle>=2.0.0,<3
-Deprecated>=1.2.7,<2
-fire>=0.3.1,<1
-jsonschema>=3.0.1,<4
-strip-hints>=0.1.8,<1
-uritemplate>=3.0.1,<4
-
-## kfp.deprecated.cli ##
-typer>=0.3.2,<1.0
-
 ## standard library backports ##
 typing-extensions>=3.7.4,<5; python_version<"3.9"

sdk/python/requirements.txt

@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with Python 3.7
-# by the following command:
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
 #
 #    pip-compile --no-emit-index-url requirements.in
 #
-absl-py==1.2.0
-    # via -r requirements.in
-attrs==22.1.0
-    # via jsonschema
 cachetools==5.2.0
     # via google-auth
 certifi==2022.6.15
@@ -18,17 +14,9 @@ certifi==2022.6.15
 charset-normalizer==2.1.1
     # via requests
 click==8.1.3
-    # via
-    #   -r requirements.in
-    #   typer
-cloudpickle==2.1.0
-    # via -r requirements.in
-deprecated==1.2.13
     # via -r requirements.in
 docstring-parser==0.14.1
     # via -r requirements.in
-fire==0.4.0
-    # via -r requirements.in
 google-api-core==2.8.2
     # via
     #   -r requirements.in
@@ -53,12 +41,6 @@ googleapis-common-protos==1.56.4
     # via google-api-core
 idna==3.3
     # via requests
-importlib-metadata==4.12.0
-    # via
-    #   click
-    #   jsonschema
-jsonschema==3.2.0
-    # via -r requirements.in
 kfp-pipeline-spec==0.1.17
     # via -r requirements.in
 kfp-server-api==2.0.0a4
@@ -79,8 +61,6 @@ pyasn1==0.4.8
     #   rsa
 pyasn1-modules==0.2.8
     # via google-auth
-pyrsistent==0.18.1
-    # via jsonschema
 python-dateutil==2.8.2
     # via
     #   kfp-server-api
@@ -104,26 +84,12 @@ rsa==4.9
     # via google-auth
 six==1.16.0
     # via
-    #   fire
     #   google-auth
-    #   jsonschema
     #   kfp-server-api
     #   kubernetes
     #   python-dateutil
-strip-hints==0.1.10
-    # via -r requirements.in
 tabulate==0.8.10
     # via -r requirements.in
-termcolor==1.1.0
-    # via fire
-typer==0.6.1
-    # via -r requirements.in
-typing-extensions==4.3.0 ; python_version < "3.9"
-    # via
-    #   -r requirements.in
-    #   importlib-metadata
-uritemplate==3.0.1
-    # via -r requirements.in
 urllib3==1.26.12
     # via
     #   kfp-server-api
@@ -131,12 +97,6 @@ urllib3==1.26.12
     #   requests
 websocket-client==1.4.0
     # via kubernetes
-wheel==0.37.1
-    # via strip-hints
-wrapt==1.14.1
-    # via deprecated
-zipp==3.8.1
-    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

test/presubmit-tests-sdk.sh

@@ -17,19 +17,19 @@ source_root=$(pwd)
 
 python3 -m pip install --upgrade pip
 python3 -m pip install coveralls==1.9.2
+python3 -m pip install $(grep 'absl-py==' sdk/python/requirements-dev.txt)
 python3 -m pip install $(grep 'docker==' sdk/python/requirements-dev.txt)
 python3 -m pip install $(grep 'pytest==' sdk/python/requirements-dev.txt)
 python3 -m pip install $(grep 'pytest-xdist==' sdk/python/requirements-dev.txt)
 python3 -m pip install $(grep 'pytest-cov==' sdk/python/requirements-dev.txt)
 python3 -m pip install --upgrade protobuf
 
-# Installing Argo CLI to lint all compiled workflows
-"${source_root}/test/install-argo-cli.sh"
-
 pushd "$source_root/sdk/python"
 python3 -m pip install -e .
 popd # Changing the current directory to the repo root for correct coverall paths
-pytest sdk/python -n auto --cov=kfp
+
+# TODO: remove deprecated dependency; then remove --ignore arg
+pytest sdk/python/kfp --ignore=sdk/python/kfp/deprecated --cov=kfp 
 
 set +x
 # export COVERALLS_REPO_TOKEN=$(gsutil cat gs://ml-pipeline-test-keys/coveralls_repo_token)

test/sample-test/Dockerfile

@@ -10,6 +10,9 @@ RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
 
 COPY ./test/sample-test/requirements.txt /python/src/github.com/kubeflow/pipelines/test/sample-test/requirements.txt
 RUN pip3 install -r /python/src/github.com/kubeflow/pipelines/test/sample-test/requirements.txt
+# TODO: remove deprecated dependency
+COPY ./sdk/python/requirements-deprecated.txt /python/src/github.com/kubeflow/pipelines/sdk/python/requirements-deprecated.txt
+RUN pip3 install -r /python/src/github.com/kubeflow/pipelines/sdk/python/requirements-deprecated.txt
 # Install KFP server API from commit.
 COPY ./backend/api/v1beta1/python_http_client /python_http_client
 # Install python client, including DSL compiler.

test/sample-test/requirements.in

@@ -8,3 +8,5 @@ kubernetes
 
 google-cloud-bigquery<3
 google-cloud-storage<2
+# TODO: remove deprecated dependency
+-r sdk/python/requirements-deprecated.txt