kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(sdk): fix resource spec test (#6979) 

* fix(sdk): fix load_test

* concat placeholder

* release notes

* fix(sdk): fix resource spec test

* release notes

* test for ontology
This commit is contained in:
Yaqi Ji 2021-11-30 11:25:31 -08:00 committed by GitHub
parent 9f743ecd6b
commit 485a07719f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 310 additions and 325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sdk/RELEASE.md
View File

@ -31,6 +31,7 @@
* Change otherwise to else in yaml [\#6952](https://github.com/kubeflow/pipelines/pull/6952)
* Avoid pydantic bug on Union type [\#6957](https://github.com/kubeflow/pipelines/pull/6957)
* Fix bug for if and concat placeholders [\#6978](https://github.com/kubeflow/pipelines/pull/6978)
* Fix bug for resourceSpec [\#6979](https://github.com/kubeflow/pipelines/pull/6979)
## Documentation Updates

4
sdk/python/kfp/v2/compiler/pipeline_spec_builder.py
View File

@ -389,9 +389,9 @@ def build_container_spec_for_task(
))
if task.container_spec.resources is not None:
container_spec.reources.cpu_limit = (
container_spec.resources.cpu_limit = (
task.container_spec.resources.cpu_limit)
container_spec.reources.memory_limit = (
container_spec.resources.memory_limit = (
task.container_spec.resources.memory_limit)
if task.container_spec.resources.accelerator_count is not None:
container_spec.resources.accelerator.CopyFrom(

10
sdk/python/kfp/v2/compiler_cli_tests/compiler_cli_tests.py
View File

@ -103,9 +103,8 @@ class CompilerCliTests(unittest.TestCase):
def test_pipeline_with_importer(self):
self._test_compile_py_to_json('pipeline_with_importer')
# TODO: re-enable the test, debug load_component_from_file error
# def test_pipeline_with_ontology(self):
# self._test_compile_py_to_json('pipeline_with_ontology')
def test_pipeline_with_ontology(self):
self._test_compile_py_to_json('pipeline_with_ontology')
def test_pipeline_with_if_placeholder(self):
self._test_compile_py_to_json('pipeline_with_if_placeholder')
@ -113,9 +112,8 @@ class CompilerCliTests(unittest.TestCase):
def test_pipeline_with_concat_placeholder(self):
self._test_compile_py_to_json('pipeline_with_concat_placeholder')
# TODO: re-enable the test, debug add_node_selector_constraint error
# def test_pipeline_with_resource_spec(self):
# self._test_compile_py_to_json('pipeline_with_resource_spec')
def test_pipeline_with_resource_spec(self):
self._test_compile_py_to_json('pipeline_with_resource_spec')
# TODO: re-enable the test, debug load_component_from_text error
# def test_pipeline_with_various_io_types(self):

300
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json
View File

@ -1,151 +1,37 @@
{
"pipelineSpec": {
"components": {
"comp-ingestion": {
"executorLabel": "exec-ingestion",
"inputDefinitions": {
"parameters": {
"input_location": {
"parameterType": "STRING"
}
}
},
"outputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
}
},
"comp-trainer": {
"executorLabel": "exec-trainer",
"inputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},
"parameters": {
"n_epochs": {
"parameterType": "NUMBER_INTEGER"
},
"optimizer": {
"parameterType": "STRING"
}
}
},
"outputDefinitions": {
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
}
}
}
},
"deploymentSpec": {
"executors": {
"exec-ingestion": {
"container": {
"args": [
"--input-location",
"{{$.inputs.parameters['input_location']}}",
"--output-examples",
"{{$.outputs.artifacts['examples'].uri}}"
],
"image": "gcr.io/my-project/my-ingestor"
}
},
"exec-trainer": {
"container": {
"args": [
"--input-examples",
"{{$.inputs.artifacts['examples'].uri}}",
"--optimizer",
"{{$.inputs.parameters['optimizer']}}",
"--n_epochs",
"{{$.inputs.parameters['n_epochs']}}",
"--output-model",
"{{$.outputs.artifacts['model'].uri}}"
],
"image": "gcr.io/my-project/my-fancy-trainer"
}
}
}
},
"pipelineInfo": {
"name": "two-step-pipeline-with-ontology"
},
"root": {
"dag": {
"tasks": {
"ingestion": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-ingestion"
},
"inputs": {
"parameters": {
"input_location": {
"componentInputParameter": "input_location"
}
}
},
"taskInfo": {
"name": "ingestion"
}
},
"trainer": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-trainer"
},
"dependentTasks": [
"ingestion"
],
"inputs": {
"artifacts": {
"examples": {
"taskOutputArtifact": {
"outputArtifactKey": "examples",
"producerTask": "ingestion"
}
}
},
"parameters": {
"n_epochs": {
"componentInputParameter": "n_epochs"
},
"optimizer": {
"componentInputParameter": "optimizer"
}
}
},
"taskInfo": {
"name": "trainer"
}
}
}
},
"components": {
"comp-ingestion": {
"executorLabel": "exec-ingestion",
"inputDefinitions": {
"parameters": {
"input_location": {
"parameterType": "STRING"
},
}
}
},
"outputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
}
},
"comp-trainer": {
"executorLabel": "exec-trainer",
"inputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},
"parameters": {
"n_epochs": {
"parameterType": "NUMBER_INTEGER"
},
@ -153,17 +39,125 @@
"parameterType": "STRING"
}
}
},
"outputDefinitions": {
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
}
}
}
},
"defaultPipelineRoot": "dummy_root",
"deploymentSpec": {
"executors": {
"exec-ingestion": {
"container": {
"args": [
"--input-location",
"{{$.inputs.parameters['input_location']}}",
"--output-examples",
"{{$.outputs.artifacts['examples'].uri}}"
],
"image": "gcr.io/my-project/my-ingestor"
}
},
"exec-trainer": {
"container": {
"args": [
"--input-examples",
"{{$.inputs.artifacts['examples'].uri}}",
"--optimizer",
"{{$.inputs.parameters['optimizer']}}",
"--n_epochs",
"{{$.inputs.parameters['n_epochs']}}",
"--output-model",
"{{$.outputs.artifacts['model'].uri}}"
],
"image": "gcr.io/my-project/my-fancy-trainer"
}
}
}
},
"pipelineInfo": {
"name": "two-step-pipeline-with-ontology"
},
"root": {
"dag": {
"tasks": {
"ingestion": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-ingestion"
},
"inputs": {
"parameters": {
"input_location": {
"componentInputParameter": "input_location"
}
}
},
"taskInfo": {
"name": "ingestion"
}
},
"trainer": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-trainer"
},
"dependentTasks": [
"ingestion"
],
"inputs": {
"artifacts": {
"examples": {
"taskOutputArtifact": {
"outputArtifactKey": "examples",
"producerTask": "ingestion"
}
}
},
"parameters": {
"n_epochs": {
"componentInputParameter": "n_epochs"
},
"optimizer": {
"componentInputParameter": "optimizer"
}
}
},
"taskInfo": {
"name": "trainer"
}
}
}
},
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.6"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",
"parameterValues": {
"input_location": "gs://test-bucket/pipeline_root",
"n_epochs": 200.0,
"optimizer": "sgd"
"inputDefinitions": {
"parameters": {
"input_location": {
"defaultValue": "gs://test-bucket/pipeline_root",
"parameterType": "STRING"
},
"n_epochs": {
"defaultValue": 200.0,
"parameterType": "NUMBER_INTEGER"
},
"optimizer": {
"defaultValue": "sgd",
"parameterType": "STRING"
}
}
}
}
},
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.9"
}

316
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json
View File

@ -1,159 +1,37 @@
{
"pipelineSpec": {
"components": {
"comp-ingestion": {
"executorLabel": "exec-ingestion",
"inputDefinitions": {
"parameters": {
"input_location": {
"parameterType": "STRING"
}
}
},
"outputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
}
},
"comp-trainer": {
"executorLabel": "exec-trainer",
"inputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},
"parameters": {
"n_epochs": {
"parameterType": "NUMBER_INTEGER"
},
"optimizer": {
"parameterType": "STRING"
}
}
},
"outputDefinitions": {
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
}
}
}
},
"deploymentSpec": {
"executors": {
"exec-ingestion": {
"container": {
"args": [
"--input-location",
"{{$.inputs.parameters['input_location']}}",
"--output-examples",
"{{$.outputs.artifacts['examples'].uri}}"
],
"image": "gcr.io/my-project/my-ingestor"
}
},
"exec-trainer": {
"container": {
"args": [
"--input-examples",
"{{$.inputs.artifacts['examples'].uri}}",
"--optimizer",
"{{$.inputs.parameters['optimizer']}}",
"--n_epochs",
"{{$.inputs.parameters['n_epochs']}}",
"--output-model",
"{{$.outputs.artifacts['model'].uri}}"
],
"image": "gcr.io/my-project/my-fancy-trainer",
"resources": {
"accelerator": {
"count": "1",
"type": "TPU_V3"
},
"cpuLimit": 4.0,
"memoryLimit": 15.032385536
}
}
}
}
},
"pipelineInfo": {
"name": "two-step-pipeline-with-resource-spec"
},
"root": {
"dag": {
"tasks": {
"ingestion": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-ingestion"
},
"inputs": {
"parameters": {
"input_location": {
"componentInputParameter": "input_location"
}
}
},
"taskInfo": {
"name": "ingestion"
}
},
"trainer": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-trainer"
},
"dependentTasks": [
"ingestion"
],
"inputs": {
"artifacts": {
"examples": {
"taskOutputArtifact": {
"outputArtifactKey": "examples",
"producerTask": "ingestion"
}
}
},
"parameters": {
"n_epochs": {
"componentInputParameter": "n_epochs"
},
"optimizer": {
"componentInputParameter": "optimizer"
}
}
},
"taskInfo": {
"name": "trainer"
}
}
}
},
"components": {
"comp-ingestion": {
"executorLabel": "exec-ingestion",
"inputDefinitions": {
"parameters": {
"input_location": {
"parameterType": "STRING"
},
}
}
},
"outputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
}
},
"comp-trainer": {
"executorLabel": "exec-trainer",
"inputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},
"parameters": {
"n_epochs": {
"parameterType": "NUMBER_INTEGER"
},
@ -161,17 +39,133 @@
"parameterType": "STRING"
}
}
},
"outputDefinitions": {
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
}
}
}
},
"defaultPipelineRoot": "dummy_root",
"deploymentSpec": {
"executors": {
"exec-ingestion": {
"container": {
"args": [
"--input-location",
"{{$.inputs.parameters['input_location']}}",
"--output-examples",
"{{$.outputs.artifacts['examples'].uri}}"
],
"image": "gcr.io/my-project/my-ingestor"
}
},
"exec-trainer": {
"container": {
"args": [
"--input-examples",
"{{$.inputs.artifacts['examples'].uri}}",
"--optimizer",
"{{$.inputs.parameters['optimizer']}}",
"--n_epochs",
"{{$.inputs.parameters['n_epochs']}}",
"--output-model",
"{{$.outputs.artifacts['model'].uri}}"
],
"image": "gcr.io/my-project/my-fancy-trainer",
"resources": {
"accelerator": {
"count": "1",
"type": "tpu-v3"
},
"cpuLimit": 4.0,
"memoryLimit": 15.032385536
}
}
}
}
},
"pipelineInfo": {
"name": "two-step-pipeline-with-resource-spec"
},
"root": {
"dag": {
"tasks": {
"ingestion": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-ingestion"
},
"inputs": {
"parameters": {
"input_location": {
"componentInputParameter": "input_location"
}
}
},
"taskInfo": {
"name": "ingestion"
}
},
"trainer": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-trainer"
},
"dependentTasks": [
"ingestion"
],
"inputs": {
"artifacts": {
"examples": {
"taskOutputArtifact": {
"outputArtifactKey": "examples",
"producerTask": "ingestion"
}
}
},
"parameters": {
"n_epochs": {
"componentInputParameter": "n_epochs"
},
"optimizer": {
"componentInputParameter": "optimizer"
}
}
},
"taskInfo": {
"name": "trainer"
}
}
}
},
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.6"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",
"parameterValues": {
"input_location": "gs://test-bucket/pipeline_root",
"n_epochs": 200.0,
"optimizer": "sgd"
"inputDefinitions": {
"parameters": {
"input_location": {
"defaultValue": "gs://test-bucket/pipeline_root",
"parameterType": "STRING"
},
"n_epochs": {
"defaultValue": 200.0,
"parameterType": "NUMBER_INTEGER"
},
"optimizer": {
"defaultValue": "sgd",
"parameterType": "STRING"
}
}
}
}
},
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.9"
}

4
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.py
View File

@ -39,9 +39,7 @@ def my_pipeline(input_location: str = 'gs://test-bucket/pipeline_root',
examples=ingestor.outputs['examples'],
optimizer=optimizer,
n_epochs=n_epochs).set_cpu_limit('4').set_memory_limit(
'14Gi').add_node_selector_constraint(
'cloud.google.com/gke-accelerator',
'tpu-v3').set_gpu_limit(1))
'14Gi').add_node_selector_constraint('tpu-v3').set_gpu_limit('1'))
if __name__ == '__main__':