kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(sdk): add default schema_version to pipeline (#6366) 

* feat(sdk): add default schema_version to pipeline

* sync api for go

* Fix tests and address comments

* Bump pipeline_spec version

* Fix v1 tests

* rebase to master

* sync api for go

* Fix tests and address comments

* Bump pipeline_spec version

* Fix v1 tests
This commit is contained in:
Yaqi Ji 2021-08-24 01:04:39 -07:00 committed by GitHub
parent 7634adbe77
commit 937cacd4ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 226 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml vendored
View File

@ -101,7 +101,7 @@ spec:
"INT"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters":
{"output_parameter_one": {"type": "INT", "path": "/tmp/outputs/output_parameter_one/data"}},
"outputArtifacts": {"output_dataset_one": {"schemaTitle": "system.Dataset",
"instanceSchema": "", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'}
"instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'}
envFrom:
- configMapRef: {name: metadata-grpc-configmap, optional: true}
image: python:3.7
@ -200,9 +200,9 @@ spec:
- {name: KFP_V2_IMAGE, value: 'python:3.7'}
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type":
"INT"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data",
"schemaTitle": "system.Dataset", "instanceSchema": ""}}, "outputParameters":
"schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters":
{}, "outputArtifacts": {"model": {"schemaTitle": "system.Model", "instanceSchema":
"", "metadataPath": "/tmp/outputs/model/data"}}}'}
"", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/model/data"}}}'}
envFrom:
- configMapRef: {name: metadata-grpc-configmap, optional: true}
image: python:3.7

6
sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml vendored
View File

@ -101,7 +101,7 @@ spec:
"INT"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters":
{"output_parameter_one": {"type": "INT", "path": "/tmp/outputs/output_parameter_one/data"}},
"outputArtifacts": {"output_dataset_one": {"schemaTitle": "system.Dataset",
"instanceSchema": "", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'}
"instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'}
envFrom:
- configMapRef: {name: metadata-grpc-configmap, optional: true}
image: python:3.7
@ -200,9 +200,9 @@ spec:
- {name: KFP_V2_IMAGE, value: 'python:3.7'}
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type":
"INT"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data",
"schemaTitle": "system.Dataset", "instanceSchema": ""}}, "outputParameters":
"schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters":
{}, "outputArtifacts": {"model": {"schemaTitle": "system.Model", "instanceSchema":
"", "metadataPath": "/tmp/outputs/model/data"}}}'}
"", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/model/data"}}}'}
envFrom:
- configMapRef: {name: metadata-grpc-configmap, optional: true}
image: python:3.7

2
sdk/python/kfp/compiler/v2_compat.py
View File

@ -153,6 +153,7 @@ def update_op(op: dsl.ContainerOp,
"metadataPath": op.input_artifact_paths[artifact_name],
"schemaTitle": spec.artifact_type.schema_title,
"instanceSchema": spec.artifact_type.instance_schema,
"schemaVersion": spec.artifact_type.schema_version,
}
runtime_info["inputArtifacts"][artifact_name] = artifact_info
@ -173,6 +174,7 @@ def update_op(op: dsl.ContainerOp,
# Type used to register output artifacts.
"schemaTitle": spec.artifact_type.schema_title,
"instanceSchema": spec.artifact_type.instance_schema,
"schemaVersion": spec.artifact_type.schema_version,
# File used to write out the registered artifact ID.
"metadataPath": op.file_outputs[artifact_name],
}

15
sdk/python/kfp/dsl/component_spec_test.py
View File

@ -64,7 +64,8 @@ class ComponentSpecTest(parameterized.TestCase):
'artifacts': {
'input1': {
'artifactType': {
'schemaTitle': 'system.Dataset'
'schemaTitle': 'system.Dataset',
'schemaVersion': '0.0.1'
}
}
},
@ -81,7 +82,8 @@ class ComponentSpecTest(parameterized.TestCase):
'artifacts': {
'output1': {
'artifactType': {
'schemaTitle': 'system.Model'
'schemaTitle': 'system.Model',
'schemaVersion': '0.0.1'
}
}
}
@ -108,7 +110,8 @@ class ComponentSpecTest(parameterized.TestCase):
'artifacts': {
'input1': {
'artifactType': {
'schemaTitle': 'system.Dataset'
'schemaTitle': 'system.Dataset',
'schemaVersion':'0.0.1'
}
}
},
@ -133,7 +136,8 @@ class ComponentSpecTest(parameterized.TestCase):
'artifacts': {
'pipelineparam--input1': {
'artifactType': {
'schemaTitle': 'system.Dataset'
'schemaTitle': 'system.Dataset',
'schemaVersion':'0.0.1'
}
}
},
@ -182,7 +186,8 @@ class ComponentSpecTest(parameterized.TestCase):
'artifacts': {
'output1': {
'artifactType': {
'schemaTitle': 'system.Dataset'
'schemaTitle': 'system.Dataset',
'schemaVersion': '0.0.1'
}
}
},

2
sdk/python/kfp/dsl/type_utils.py
View File

@ -27,4 +27,4 @@ is_parameter_type = type_utils.is_parameter_type
get_artifact_type_schema = type_utils.type_utils.get_artifact_type_schema
get_parameter_type = type_utils.get_parameter_type
get_parameter_type_field_name = type_utils.get_parameter_type_field_name
get_input_artifact_type_schema = type_utils.type_utils.get_input_artifact_type_schema
get_input_artifact_type_schema = type_utils.type_utils.get_input_artifact_type_schema

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/experimental_v2_component.json
View File

@ -123,7 +123,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

15
sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json
View File

@ -14,12 +14,14 @@
"artifacts": {
"output_dataset_one": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
},
"output_dataset_two_path": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},
@ -45,12 +47,14 @@
"artifacts": {
"dataset_one_path": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
},
"dataset_two": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},
@ -76,7 +80,8 @@
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
}

15
sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json
View File

@ -57,7 +57,8 @@
"artifacts": {
"Output": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
@ -69,7 +70,8 @@
"artifacts": {
"artifact": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
@ -78,12 +80,14 @@
"artifacts": {
"metrics": {
"artifactType": {
"schemaTitle": "system.Metrics"
"schemaTitle": "system.Metrics",
"schemaVersion": "0.0.1"
}
},
"model": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
},
@ -316,7 +320,8 @@
"artifacts": {
"output-named-tuple-metrics": {
"artifactType": {
"schemaTitle": "system.Metrics"
"schemaTitle": "system.Metrics",
"schemaVersion": "0.0.1"
}
}
}

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json
View File

@ -152,7 +152,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json
View File

@ -57,7 +57,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json
View File

@ -275,7 +275,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

24
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json
View File

@ -72,7 +72,8 @@
"artifacts": {
"artifact": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
@ -91,7 +92,8 @@
"artifacts": {
"artifact": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
@ -103,7 +105,8 @@
"artifacts": {
"dataset": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
@ -112,7 +115,8 @@
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
},
@ -129,7 +133,8 @@
"artifacts": {
"dataset": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
@ -138,7 +143,8 @@
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
},
@ -160,7 +166,8 @@
}
},
"typeSchema": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},
@ -170,7 +177,8 @@
"runtimeParameter": "uri"
},
"typeSchema": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_output.json
View File

@ -226,7 +226,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_parameter.json
View File

@ -191,7 +191,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_static.json
View File

@ -174,7 +174,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json
View File

@ -678,7 +678,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

15
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json
View File

@ -40,7 +40,8 @@
"artifacts": {
"output-metrics-2-metrics": {
"artifactType": {
"schemaTitle": "system.Metrics"
"schemaTitle": "system.Metrics",
"schemaVersion": "0.0.1"
}
}
}
@ -52,7 +53,8 @@
"artifacts": {
"metrics": {
"artifactType": {
"schemaTitle": "system.Metrics"
"schemaTitle": "system.Metrics",
"schemaVersion": "0.0.1"
}
}
}
@ -64,7 +66,8 @@
"artifacts": {
"metrics": {
"artifactType": {
"schemaTitle": "system.Metrics"
"schemaTitle": "system.Metrics",
"schemaVersion": "0.0.1"
}
}
}
@ -172,12 +175,14 @@
"artifacts": {
"output-metrics-2-metrics": {
"artifactType": {
"schemaTitle": "system.Metrics"
"schemaTitle": "system.Metrics",
"schemaVersion": "0.0.1"
}
},
"output-metrics-metrics": {
"artifactType": {
"schemaTitle": "system.Metrics"
"schemaTitle": "system.Metrics",
"schemaVersion": "0.0.1"
}
}
}

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json
View File

@ -452,7 +452,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json
View File

@ -544,7 +544,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json
View File

@ -142,7 +142,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

11
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json
View File

@ -14,7 +14,8 @@
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
@ -26,7 +27,8 @@
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},
@ -43,7 +45,8 @@
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
}
@ -153,7 +156,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json
View File

@ -251,7 +251,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

11
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json
View File

@ -14,7 +14,8 @@
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
}
@ -26,7 +27,8 @@
"artifacts": {
"examples": {
"artifactType": {
"schemaTitle": "system.Dataset"
"schemaTitle": "system.Dataset",
"schemaVersion": "0.0.1"
}
}
},
@ -43,7 +45,8 @@
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
}
}
@ -161,7 +164,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

2
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json
View File

@ -208,7 +208,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

44
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json
View File

@ -7,37 +7,44 @@
"artifacts": {
"input_b": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
},
"input_c": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"input_d": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
},
"input_e": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"input_f": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"input_g": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"input_h": {
"artifactType": {
"schemaTitle": "system.HTML"
"schemaTitle": "system.HTML",
"schemaVersion": "0.0.1"
}
}
},
@ -70,37 +77,44 @@
"artifacts": {
"output_2": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
},
"output_3": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"output_4": {
"artifactType": {
"schemaTitle": "system.Model"
"schemaTitle": "system.Model",
"schemaVersion": "0.0.1"
}
},
"output_5": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"output_6": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"output_7": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"output_8": {
"artifactType": {
"schemaTitle": "system.HTML"
"schemaTitle": "system.HTML",
"schemaVersion": "0.0.1"
}
}
},
@ -272,7 +286,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

8
sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json
View File

@ -7,7 +7,8 @@
"artifacts": {
"input_gcs_path": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -26,7 +27,8 @@
"artifacts": {
"output_gcs_path": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -119,7 +121,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.6"
"sdkVersion": "kfp-1.7.0"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

63
sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json
View File

@ -23,7 +23,8 @@
"artifacts": {
"Table": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -35,7 +36,8 @@
"artifacts": {
"data": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -44,7 +46,8 @@
"artifacts": {
"output_data": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -56,12 +59,14 @@
"artifacts": {
"data": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"model": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
},
@ -75,7 +80,8 @@
"artifacts": {
"predictions": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -87,12 +93,14 @@
"artifacts": {
"data": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"model": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
},
@ -106,7 +114,8 @@
"artifacts": {
"predictions": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -118,12 +127,14 @@
"artifacts": {
"data": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"model": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
},
@ -137,7 +148,8 @@
"artifacts": {
"predictions": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -149,12 +161,14 @@
"artifacts": {
"data": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"model": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
},
@ -168,7 +182,8 @@
"artifacts": {
"predictions": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -180,7 +195,8 @@
"artifacts": {
"training_data": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
},
@ -212,12 +228,14 @@
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"model_config": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}
@ -229,7 +247,8 @@
"artifacts": {
"training_data": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
},
@ -261,12 +280,14 @@
"artifacts": {
"model": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
},
"model_config": {
"artifactType": {
"schemaTitle": "system.Artifact"
"schemaTitle": "system.Artifact",
"schemaVersion": "0.0.1"
}
}
}

3
sdk/python/kfp/v2/components/types/artifact_types.py
View File

@ -36,6 +36,7 @@ class Artifact(object):
metadata related to this artifact.
"""
TYPE_NAME = 'system.Artifact'
VERSION = '0.0.1'
def __init__(self,
name: Optional[str] = None,
@ -448,4 +449,4 @@ def create_runtime_artifact(runtime_artifact: Dict) -> Artifact:
uri=runtime_artifact.get('uri', ''),
name=runtime_artifact.get('name', ''),
metadata=runtime_artifact.get('metadata', {}),
)
)

3
sdk/python/kfp/v2/components/types/type_utils.py
View File

@ -91,7 +91,8 @@ def get_artifact_type_schema(
artifact_class = artifact_class_or_type_name
return pipeline_spec_pb2.ArtifactTypeSchema(
schema_title=artifact_class.TYPE_NAME)
schema_title=artifact_class.TYPE_NAME,
schema_version=artifact_class.VERSION)
def get_parameter_type(

42
sdk/python/kfp/v2/components/types/type_utils_test.py
View File

@ -62,87 +62,101 @@ class TypeUtilsTest(parameterized.TestCase):
'artifact_class_or_type_name':
'Model',
'expected_result':
pb.ArtifactTypeSchema(schema_title='system.Model')
pb.ArtifactTypeSchema(schema_title='system.Model',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
artifact_types.Model,
'expected_result':
pb.ArtifactTypeSchema(schema_title='system.Model')
pb.ArtifactTypeSchema(schema_title='system.Model',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
'Dataset',
'expected_result':
pb.ArtifactTypeSchema(schema_title='system.Dataset')
pb.ArtifactTypeSchema(schema_title='system.Dataset',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
artifact_types.Dataset,
'expected_result':
pb.ArtifactTypeSchema(schema_title='system.Dataset')
pb.ArtifactTypeSchema(schema_title='system.Dataset',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
'Metrics',
'expected_result':
pb.ArtifactTypeSchema(schema_title='system.Metrics')
pb.ArtifactTypeSchema(schema_title='system.Metrics',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
artifact_types.Metrics,
'expected_result':
pb.ArtifactTypeSchema(schema_title='system.Metrics')
pb.ArtifactTypeSchema(schema_title='system.Metrics',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
'ClassificationMetrics',
'expected_result':
pb.ArtifactTypeSchema(
schema_title='system.ClassificationMetrics')
schema_title='system.ClassificationMetrics',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
artifact_types.ClassificationMetrics,
'expected_result':
pb.ArtifactTypeSchema(
schema_title='system.ClassificationMetrics')
schema_title='system.ClassificationMetrics',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
'SlicedClassificationMetrics',
'expected_result':
pb.ArtifactTypeSchema(
schema_title='system.SlicedClassificationMetrics')
schema_title='system.SlicedClassificationMetrics',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
artifact_types.SlicedClassificationMetrics,
'expected_result':
pb.ArtifactTypeSchema(
schema_title='system.SlicedClassificationMetrics')
schema_title='system.SlicedClassificationMetrics',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
'arbitrary name',
'expected_result':
pb.ArtifactTypeSchema(schema_title='system.Artifact')
pb.ArtifactTypeSchema(schema_title='system.Artifact',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
_ArbitraryClass,
'expected_result':
pb.ArtifactTypeSchema(schema_title='system.Artifact')
pb.ArtifactTypeSchema(schema_title='system.Artifact',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name': artifact_types.HTML,
'expected_result': pb.ArtifactTypeSchema(schema_title='system.HTML')
'expected_result': pb.ArtifactTypeSchema(schema_title='system.HTML',
schema_version='0.0.1')
},
{
'artifact_class_or_type_name':
artifact_types.Markdown,
'expected_result':
pb.ArtifactTypeSchema(schema_title='system.Markdown')
pb.ArtifactTypeSchema(schema_title='system.Markdown',
schema_version='0.0.1')
},
)
def test_get_artifact_type_schema(self, artifact_class_or_type_name,

2
sdk/python/requirements.in
View File

@ -29,7 +29,7 @@ click>=7.1.1,<8
# kfp.v2
absl-py>=0.9,<=0.11
kfp-pipeline-spec>=0.1.8,<0.2.0
kfp-pipeline-spec>=0.1.9,<0.2.0
fire>=0.3.1,<1
google-api-python-client>=1.7.8,<2
dataclasses>=0.8,<1; python_version<"3.7"

20
sdk/python/requirements.txt
View File

@ -1,5 +1,5 @@
#
# This file is autogenerated by pip-compile
# This file is autogenerated by pip-compile with python 3.9
# To update, run:
#
# pip-compile requirements.in
@ -35,8 +35,6 @@ google-api-core==1.26.0
# google-cloud-core
google-api-python-client==1.12.8
# via -r requirements.in
google-auth-httplib2==0.1.0
# via google-api-python-client
google-auth==1.27.0
# via
# -r requirements.in
@ -46,6 +44,8 @@ google-auth==1.27.0
# google-cloud-core
# google-cloud-storage
# kubernetes
google-auth-httplib2==0.1.0
# via google-api-python-client
google-cloud-core==1.6.0
# via google-cloud-storage
google-cloud-storage==1.36.1
@ -64,7 +64,7 @@ idna==2.10
# via requests
jsonschema==3.2.0
# via -r requirements.in
kfp-pipeline-spec==0.1.8
kfp-pipeline-spec==0.1.9
# via -r requirements.in
kfp-server-api==1.3.0
# via -r requirements.in
@ -79,12 +79,12 @@ protobuf==3.15.2
# google-api-core
# googleapis-common-protos
# kfp-pipeline-spec
pyasn1-modules==0.2.8
# via google-auth
pyasn1==0.4.8
# via
# pyasn1-modules
# rsa
pyasn1-modules==0.2.8
# via google-auth
pycparser==2.20
# via cffi
pyparsing==2.4.7
@ -103,10 +103,6 @@ pyyaml==5.4.1
# via
# -r requirements.in
# kubernetes
requests-oauthlib==1.3.0
# via kubernetes
requests-toolbelt==0.9.1
# via -r requirements.in
requests==2.25.1
# via
# google-api-core
@ -114,6 +110,10 @@ requests==2.25.1
# kubernetes
# requests-oauthlib
# requests-toolbelt
requests-oauthlib==1.3.0
# via kubernetes
requests-toolbelt==0.9.1
# via -r requirements.in
rsa==4.7.2
# via google-auth
six==1.15.0

2
sdk/python/setup.py
View File

@ -46,7 +46,7 @@ REQUIRES = [
'Deprecated>=1.2.7,<2',
'strip-hints>=0.1.8,<1',
'docstring-parser>=0.7.3,<1',
'kfp-pipeline-spec>=0.1.8,<0.2.0',
'kfp-pipeline-spec>=0.1.9,<0.2.0',
'fire>=0.3.1,<1',
'protobuf>=3.13.0,<4',
# Standard library backports

10
sdk/python/tests/compiler/compiler_tests.py
View File

@ -487,33 +487,33 @@ class TestCompiler(parameterized.TestCase):
template = name_to_template[main_dag_tasks[0]['template']]
self.assertEqual(template['podSpecPatch'], '{"containers": [{"name": "main", "resources": {"requests": {"cpu": "{{inputs.parameters.memory}}"}}}]}')
def test_py_runtime_gpu_request(self):
"""Test GPU request."""
def my_pipeline(nbr_gpus: int, gpu_vendor: str):
some_op().set_gpu_limit(nbr_gpus, gpu_vendor)
workflow = kfp.compiler.Compiler()._create_workflow(my_pipeline)
name_to_template = {template['name']: template for template in workflow['spec']['templates']}
main_dag_tasks = name_to_template[workflow['spec']['entrypoint']]['dag']['tasks']
template = name_to_template[main_dag_tasks[0]['template']]
self.assertEqual(template['podSpecPatch'], '{"containers": [{"name": "main", "resources": {"limits": {"{{inputs.parameters.gpu_vendor}}": "{{inputs.parameters.nbr_gpus}}"}}}]}')
def test_py_runtime_node_selection(self):
"""Test node selection request."""
def my_pipeline(constrain_type: str, constrain_value: str):
some_op().add_node_selector_constraint(constrain_type, constrain_value)
workflow = kfp.compiler.Compiler()._create_workflow(my_pipeline)
name_to_template = {template['name']: template for template in workflow['spec']['templates']}
main_dag_tasks = name_to_template[workflow['spec']['entrypoint']]['dag']['tasks']
template = name_to_template[main_dag_tasks[0]['template']]
self.assertEqual(template['podSpecPatch'], '{"nodeSelector": [{"{{inputs.parameters.constrain_type}}": "{{inputs.parameters.constrain_value}}"}]}')
def test_py_retry_policy_invalid(self):
def my_pipeline():

18
sdk/python/tests/compiler/testdata/uri_artifacts.yaml vendored
View File

@ -7,8 +7,8 @@ metadata:
pipelines.kubeflow.org/pipeline_compilation_time: '2021-08-07T00:27:52.329848'
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "Hello world!",
"name": "text", "optional": true, "type": "String"}, {"default": "gs://my-bucket/my-output-dir",
"name": "pipeline-root"}, {"default": "pipeline/uri-artifact-pipeline",
"name": "pipeline-name"}], "name": "uri-artifact-pipeline"}'
"name": "pipeline-root"}, {"default": "pipeline/uri-artifact-pipeline", "name":
"pipeline-name"}], "name": "uri-artifact-pipeline"}'
pipelines.kubeflow.org/v2_pipeline: "true"
labels:
pipelines.kubeflow.org/v2_pipeline: "true"
@ -154,8 +154,8 @@ spec:
- {name: KFP_V2_IMAGE, value: 'google/cloud-sdk:slim'}
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts":
{"input_gcs_path": {"metadataPath": "/tmp/inputs/input_gcs_path/data", "schemaTitle":
"system.Artifact", "instanceSchema": ""}}, "outputParameters": {}, "outputArtifacts":
{}}'}
"system.Artifact", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters":
{}, "outputArtifacts": {}}'}
envFrom:
- configMapRef: {name: metadata-grpc-configmap, optional: true}
image: google/cloud-sdk:slim
@ -221,8 +221,8 @@ spec:
- {name: KFP_V2_IMAGE, value: 'google/cloud-sdk:slim'}
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts":
{"input_gcs_path": {"metadataPath": "/tmp/inputs/input_gcs_path/data", "schemaTitle":
"system.Artifact", "instanceSchema": ""}}, "outputParameters": {}, "outputArtifacts":
{}}'}
"system.Artifact", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters":
{}, "outputArtifacts": {}}'}
envFrom:
- configMapRef: {name: metadata-grpc-configmap, optional: true}
image: google/cloud-sdk:slim
@ -288,8 +288,8 @@ spec:
- {name: KFP_V2_IMAGE, value: 'google/cloud-sdk:slim'}
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts":
{"input_gcs_path": {"metadataPath": "/tmp/inputs/input_gcs_path/data", "schemaTitle":
"system.Artifact", "instanceSchema": ""}}, "outputParameters": {}, "outputArtifacts":
{}}'}
"system.Artifact", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters":
{}, "outputArtifacts": {}}'}
envFrom:
- configMapRef: {name: metadata-grpc-configmap, optional: true}
image: google/cloud-sdk:slim
@ -408,7 +408,7 @@ spec:
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"text": {"type":
"STRING"}}, "inputArtifacts": {}, "outputParameters": {}, "outputArtifacts":
{"output_gcs_path": {"schemaTitle": "system.Artifact", "instanceSchema":
"", "metadataPath": "/tmp/outputs/output_gcs_path/data"}}}'}
"", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_gcs_path/data"}}}'}
envFrom:
- configMapRef: {name: metadata-grpc-configmap, optional: true}
image: google/cloud-sdk:slim