test: refactor v2 sample test, move test samples to samples/test folder (#5395)
* refactor: sample test util * fix * fix * refactor v2 sample test * fix
This commit is contained in:
Yuan (Bob) Gong
GitHub
parent
1e32f01dc4
commit
14b447a07c
|
|
@ -0,0 +1,6 @@
|
|||
# Test Samples
|
||||
|
||||
These samples are built for testing purposes only.
|
||||
|
||||
The [config.yaml](./config.yaml) holds test config for kubeflow-pipelines-samples-v2 test.
|
||||
Refer to [V2 samples test documentation](https://github.com/kubeflow/pipelines/tree/master/v2/test) for more details.
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# kubeflow-pipelines-samples-v2 test config
|
||||
# Documentation: https://github.com/kubeflow/pipelines/tree/master/v2/test
|
||||
#
|
||||
#### config format
|
||||
#
|
||||
# Each item in the list corresponds to test for one sample.
|
||||
#
|
||||
# The field `path` corresponds to the test's python module path
|
||||
# e.g. if folder path is `samples/test/fail_test.py`, then module path is
|
||||
# `samples.test.fail_test`.
|
||||
- name: fail
|
||||
path: samples.test.fail_test
|
||||
- name: two_step
|
||||
path: samples.test.two_step_test
|
||||
|
|
@ -11,4 +11,15 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Fail pipeline."""
|
||||
|
||||
from .fail import fail_pipeline
|
||||
from .util import run_pipeline_func
|
||||
|
||||
|
||||
def verify(run, run_id: str):
|
||||
assert run.status == 'Failed'
|
||||
# TODO(Bobgy): verify MLMD status
|
||||
|
||||
|
||||
run_pipeline_func(pipeline_func=fail_pipeline, verify_func=verify)
|
||||
|
|
@ -11,12 +11,15 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Two step v2-compatible pipeline."""
|
||||
|
||||
# sample test config
|
||||
# path corresponds to python module path
|
||||
# e.g. if folder path is `v2/test/samples/fail_test.py`, then module path is
|
||||
# `v2.test.samples.fail_test`.
|
||||
- name: fail
|
||||
path: v2.test.samples.fail_test
|
||||
- name: two_step
|
||||
path: v2.test.samples.two_step_test
|
||||
from .two_step import two_step_pipeline
|
||||
from .util import run_pipeline_func
|
||||
|
||||
|
||||
def verify(run, run_id: str):
|
||||
assert run.status == 'Succeeded'
|
||||
# TODO(Bobgy): verify MLMD status
|
||||
|
||||
|
||||
run_pipeline_func(pipeline_func=two_step_pipeline, verify_func=verify)
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
import kfp
|
||||
|
||||
MINUTE = 60
|
||||
|
||||
|
||||
def default_verify_func(run_id, run):
|
||||
assert run.status == 'Succeeded'
|
||||
|
||||
|
||||
def run_pipeline_func(
|
||||
pipeline_func,
|
||||
verify_func=default_verify_func,
|
||||
mode=kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE
|
||||
):
|
||||
"""Run a pipeline function and wait for its result.
|
||||
|
||||
:param pipeline_func: pipeline function to run
|
||||
:type pipeline_func: function
|
||||
"""
|
||||
|
||||
def main(
|
||||
output_directory: str, # example gs://your-bucket/path/to/workdir
|
||||
host: str = 'http://ml-pipeline:8888',
|
||||
launcher_image: 'URI' = None,
|
||||
experiment: str = 'v2_sample_test_samples',
|
||||
):
|
||||
"""Test file CLI entrypoint used by Fire.
|
||||
|
||||
:param output_directory: pipeline output directory that holds intermediate artifacts.
|
||||
:type output_directory: str
|
||||
:param launcher_image: override launcher image, only used in V2_COMPATIBLE mode
|
||||
:type launcher_image: URI, optional
|
||||
:param experiment: experiment the run is added to, defaults to 'v2_sample_test_samples'
|
||||
:type experiment: str, optional
|
||||
"""
|
||||
client = kfp.Client(host=host)
|
||||
run_result = client.create_run_from_pipeline_func(
|
||||
pipeline_func,
|
||||
mode=mode,
|
||||
arguments={kfp.dsl.ROOT_PARAMETER_NAME: output_directory},
|
||||
launcher_image=launcher_image,
|
||||
experiment_name=experiment,
|
||||
)
|
||||
print("Run details page URL:")
|
||||
print(f"{host}/#/runs/details/{run_result.run_id}")
|
||||
run_response = run_result.wait_for_run_completion(10 * MINUTE)
|
||||
run = run_response.run
|
||||
from pprint import pprint
|
||||
pprint(run_response.run)
|
||||
print("Run details page URL:")
|
||||
print(f"{host}/#/runs/details/{run_result.run_id}")
|
||||
verify_func(run_id=run_result.run_id, run=run)
|
||||
|
||||
import fire
|
||||
fire.Fire(main)
|
||||
|
|
@ -3,9 +3,17 @@
|
|||
Note, the sample test only runs on Google Cloud at the moment. Welcome
|
||||
contribution if you want to adapt it to other platforms.
|
||||
|
||||
Quick Links:
|
||||
|
||||
* [prowjob config](https://github.com/GoogleCloudPlatform/oss-test-infra/blob/48b09567c8df28fab2d3f2fb6df86defa12207fb/prow/prowjobs/kubeflow/pipelines/kubeflow-pipelines-presubmits.yaml#L184-L192)
|
||||
* [past prow jobs](https://oss-prow.knative.dev/job-history/gs/oss-prow/pr-logs/directory/kubeflow-pipelines-samples-v2)
|
||||
* [sample test config](../../samples/test/config.yaml)
|
||||
* [KFP test cluster hostname](https://github.com/kubeflow/testing/blob/master/test-infra/kfp/endpoint)
|
||||
* [Infra as Code configuration for kfp-ci project](https://github.com/kubeflow/testing/tree/master/test-infra/kfp).
|
||||
|
||||
## How to access the KFP UI running these tests?
|
||||
|
||||
kubeflow-pipelines-sample-v2 test pipeline runs on `kfp-standalone-1` cluster,
|
||||
kubeflow-pipelines-sample-v2 test pipeline runs on [kfp-standalone-1 cluster](https://console.cloud.google.com/kubernetes/clusters/details/us-central1/kfp-standalone-1/details?folder=&organizationId=&project=kfp-ci),
|
||||
`kfp-ci` project, `kubeflow.org` organization.
|
||||
|
||||
The test script prints KFP host URL in logs. You need to have permission to
|
||||
|
|
@ -21,22 +29,24 @@ Contact @Bobgy if you have such a need.
|
|||
|
||||
## How to run the entire sample test suite in your own KFP?
|
||||
|
||||
You need to create an `.env` file in this folder and add the following config to
|
||||
it:
|
||||
You need to create an `.env` file in this folder and add the following config:
|
||||
|
||||
```env
|
||||
GCS_ROOT=gs://path/to/sample/test/workingdir
|
||||
GCR_ROOT=gcr.io/path/to/sample/test/container/registry
|
||||
HOST=https://<your KFP host>
|
||||
HOST=https://your.kfp.hostname.com
|
||||
```
|
||||
|
||||
You need to login locally to allow uploading source folder to the GCS_ROOT:
|
||||
|
||||
```bash
|
||||
gcloud auth application-default login
|
||||
# Or use the following to login both gcloud and application default
|
||||
# at the same time.
|
||||
gcloud auth login --update-adc
|
||||
```
|
||||
|
||||
Your KFP cluster should have permission the GCS_ROOT and the GCR_ROOT.
|
||||
Your KFP cluster should have permission for the GCS_ROOT and the GCR_ROOT.
|
||||
|
||||
Run sample test by:
|
||||
|
||||
|
|
@ -55,19 +65,20 @@ For why the caveat exists, refer to context rule in [Makefile](./Makefile).
|
|||
|
||||
```bash
|
||||
cd ${REPO_ROOT}
|
||||
python -m v2.test.samples.<your_test_name> --host <your-KFP-host> --output_directory gs://your-bucket/path/to/output/dir
|
||||
# or to see all available command line arguments
|
||||
python -m v2.test.samples.<your_test_name> --help
|
||||
# if you have a sample test at samples/path/to/your/sample_test.py
|
||||
python -m samples.path.to.your.sample_test --host https://your.KFP.host --output_directory gs://your-bucket/path/to/output/dir
|
||||
# or to look at command help
|
||||
python -m samples.path.to.your.sample_test --help
|
||||
```
|
||||
|
||||
## How to add a sample to this test infra?
|
||||
## How to add a sample to this sample test?
|
||||
|
||||
Edit [samples_config.yaml](./samples_config.yaml) and add your own sample.
|
||||
You can also add samples not in the `v2` folder.
|
||||
Edit [samples/test/config.yaml](../../samples/test/config.yaml) and add your own sample.
|
||||
You can also add other samples not in the `samples/test` folder.
|
||||
|
||||
Your sample needs to conform to the standard interface in
|
||||
Your sample test needs to conform to the standard interface in
|
||||
[components/run_sample.yaml](components/run_sample.yaml). You can refer to
|
||||
existing [samples](samples) for how to implement the interface.
|
||||
existing [sample tests](../../samples/test) for how to implement the interface.
|
||||
|
||||
## Implementation Details
|
||||
|
||||
|
|
@ -81,8 +92,3 @@ the following steps:
|
|||
1. builds needed images
|
||||
2. compiles, creates and waits for sub sample KFP pipelines
|
||||
3. verifies execution result of sample pipelines
|
||||
|
||||
## How is test infra set up?
|
||||
|
||||
* [Prow test config for this presubmit test](https://github.com/GoogleCloudPlatform/oss-test-infra/blob/a4dda24bcc0afc811c4cda9671c2bc48da499cef/prow/prowjobs/kubeflow/pipelines/kubeflow-pipelines-presubmits.yaml#L184-L192).
|
||||
* [Infra as Code configuration for kfp-ci project](https://github.com/kubeflow/testing/tree/master/test-infra/kfp).
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ implementation:
|
|||
launcher_image="$4"
|
||||
|
||||
python3 -m "$sample_path" \
|
||||
--pipeline_root "$root/$name" \
|
||||
--output_directory "$root/$name" \
|
||||
--host "$host" \
|
||||
--launcher_image "$launcher_image"
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
# install kfp sdk from local path
|
||||
-e ../../sdk/python
|
||||
|
|
|
|||
|
|
@ -14,9 +14,12 @@
|
|||
|
||||
# %%
|
||||
import yaml
|
||||
import os
|
||||
|
||||
REPO_ROOT = os.path.join('..', '..')
|
||||
SAMPLES_CONFIG_PATH = os.path.join(REPO_ROOT, 'samples', 'test', 'config.yaml')
|
||||
SAMPLES_CONFIG = None
|
||||
with open('samples_config.yaml', 'r') as stream:
|
||||
with open(SAMPLES_CONFIG_PATH, 'r') as stream:
|
||||
SAMPLES_CONFIG = yaml.safe_load(stream)
|
||||
|
||||
import kfp
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
# Test Samples
|
||||
|
||||
These samples are built for testing purposes only.
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Fail pipeline."""
|
||||
|
||||
import kfp
|
||||
from .fail import fail_pipeline
|
||||
|
||||
MINUTE = 60 # seconds
|
||||
|
||||
|
||||
def main(
|
||||
pipeline_root: str = 'gs://your-bucket/path/to/workdir',
|
||||
host: str = 'http://ml-pipeline:8888',
|
||||
launcher_image: 'URI' = None,
|
||||
experiment: str = 'v2_sample_test_samples'
|
||||
):
|
||||
client = kfp.Client(host=host)
|
||||
run_result = client.create_run_from_pipeline_func(
|
||||
fail_pipeline,
|
||||
mode=kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE,
|
||||
arguments={kfp.dsl.ROOT_PARAMETER_NAME: pipeline_root},
|
||||
launcher_image=launcher_image,
|
||||
experiment_name=experiment,
|
||||
)
|
||||
print("Run details page URL:")
|
||||
print(f"{host}/#/runs/details/{run_result.run_id}")
|
||||
run_response = run_result.wait_for_run_completion(timeout=10 * MINUTE)
|
||||
run = run_response.run
|
||||
from pprint import pprint
|
||||
pprint(run_response.run)
|
||||
print("Run details page URL:")
|
||||
print(f"{host}/#/runs/details/{run.id}")
|
||||
assert run.status == 'Failed'
|
||||
# TODO: add more MLMD verification
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import fire
|
||||
fire.Fire(main)
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Two step v2-compatible pipeline."""
|
||||
|
||||
import kfp
|
||||
from .two_step import two_step_pipeline
|
||||
|
||||
MINUTE = 60
|
||||
|
||||
|
||||
def main(
|
||||
pipeline_root: str = 'gs://your-bucket/path/to/workdir',
|
||||
host: str = 'http://ml-pipeline:8888',
|
||||
launcher_image: 'URI' = None,
|
||||
experiment: str = 'v2_sample_test_samples',
|
||||
):
|
||||
client = kfp.Client(host=host)
|
||||
run_result = client.create_run_from_pipeline_func(
|
||||
two_step_pipeline,
|
||||
mode=kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE,
|
||||
arguments={kfp.dsl.ROOT_PARAMETER_NAME: pipeline_root},
|
||||
launcher_image=launcher_image,
|
||||
experiment_name=experiment,
|
||||
)
|
||||
print("Run details page URL:")
|
||||
print(f"{host}/#/runs/details/{run_result.run_id}")
|
||||
run_response = run_result.wait_for_run_completion(10 * MINUTE)
|
||||
run = run_response.run
|
||||
from pprint import pprint
|
||||
pprint(run_response.run)
|
||||
print("Run details page URL:")
|
||||
print(f"{host}/#/runs/details/{run_result.run_id}")
|
||||
assert run.status == 'Succeeded'
|
||||
# TODO(Bobgy): print debug info
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import fire
|
||||
fire.Fire(main)
|
||||
Loading…
Reference in New Issue