kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(sdk): fix load_test (#6978) 

* fix(sdk): fix load_test

* concat placeholder

* release notes
This commit is contained in:
Yaqi Ji 2021-11-29 15:00:30 -08:00 committed by GitHub
parent 9cb0e7127c
commit 0842934bae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 124 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sdk/RELEASE.md
View File

@ -30,6 +30,8 @@
* Depends on `google-auth>=1.6.1,<3` [\#6939](https://github.com/kubeflow/pipelines/pull/6939)
* Change otherwise to else in yaml [\#6952](https://github.com/kubeflow/pipelines/pull/6952)
* Avoid pydantic bug on Union type [\#6957](https://github.com/kubeflow/pipelines/pull/6957)
* Fix bug for if and concat placeholders [\#6978](https://github.com/kubeflow/pipelines/pull/6978)
## Documentation Updates

12
sdk/python/kfp/v2/compiler_cli_tests/compiler_cli_tests.py
View File

@ -107,15 +107,13 @@ class CompilerCliTests(unittest.TestCase):
# def test_pipeline_with_ontology(self):
# self._test_compile_py_to_json('pipeline_with_ontology')
# TODO: re-enable the test, debug load_component_from_file error
# def test_pipeline_with_if_placeholder(self):
# self._test_compile_py_to_json('pipeline_with_if_placeholder')
def test_pipeline_with_if_placeholder(self):
self._test_compile_py_to_json('pipeline_with_if_placeholder')
# TODO: re-enable the test, debug load_component_from_file error
# def test_pipeline_with_concat_placeholder(self):
# self._test_compile_py_to_json('pipeline_with_concat_placeholder')
def test_pipeline_with_concat_placeholder(self):
self._test_compile_py_to_json('pipeline_with_concat_placeholder')
# TODO: re-enable the test, debug load_component_from_file error
# TODO: re-enable the test, debug add_node_selector_constraint error
# def test_pipeline_with_resource_spec(self):
# self._test_compile_py_to_json('pipeline_with_resource_spec')

94
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json
View File

@ -1,63 +1,59 @@
{
"pipelineSpec": {
"components": {
"comp-component-with-concat-placeholder": {
"executorLabel": "exec-component-with-concat-placeholder",
"inputDefinitions": {
"parameters": {
"input_prefix": {
"parameterType": "STRING"
}
"components": {
"comp-component-with-concat-placeholder": {
"executorLabel": "exec-component-with-concat-placeholder",
"inputDefinitions": {
"parameters": {
"input_prefix": {
"parameterType": "STRING"
}
}
}
},
"deploymentSpec": {
"executors": {
"exec-component-with-concat-placeholder": {
"container": {
"args": [
"--arg0",
"{{$.inputs.parameters['input_prefix']}}some value"
],
"image": "gcr.io/my-project/my-image"
}
}
},
"defaultPipelineRoot": "dummy_root",
"deploymentSpec": {
"executors": {
"exec-component-with-concat-placeholder": {
"container": {
"args": [
"--arg0",
"{{$.inputs.parameters['input_prefix']}}some value"
],
"image": "gcr.io/my-project/my-image"
}
}
},
"pipelineInfo": {
"name": "one-step-pipeline-with-concat-placeholder"
},
"root": {
"dag": {
"tasks": {
"component-with-concat-placeholder": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-concat-placeholder"
},
"inputs": {
"parameters": {
"input_prefix": {
"runtimeValue": {
"constant": "some prefix:"
}
}
},
"pipelineInfo": {
"name": "one-step-pipeline-with-concat-placeholder"
},
"root": {
"dag": {
"tasks": {
"component-with-concat-placeholder": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-concat-placeholder"
},
"inputs": {
"parameters": {
"input_prefix": {
"runtimeValue": {
"constant": "some prefix:"
}
}
},
"taskInfo": {
"name": "component-with-concat-placeholder"
}
},
"taskInfo": {
"name": "component-with-concat-placeholder"
}
}
}
},
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.6"
}
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"
}
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.9"
}

142
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json
View File

@ -1,84 +1,80 @@
{
"pipelineSpec": {
"components": {
"comp-component-with-optional-inputs": {
"executorLabel": "exec-component-with-optional-inputs",
"inputDefinitions": {
"parameters": {
"optional_input_1": {
"parameterType": "STRING"
},
"required_input": {
"parameterType": "STRING"
}
}
}
}
},
"deploymentSpec": {
"executors": {
"exec-component-with-optional-inputs": {
"container": {
"args": [
"--arg0",
"{{$.inputs.parameters['required_input']}}",
"--arg1",
"{{$.inputs.parameters['optional_input_1']}}",
"--arg3",
"default value"
],
"image": "gcr.io/my-project/my-image"
}
}
}
},
"pipelineInfo": {
"name": "one-step-pipeline-with-if-placeholder"
},
"root": {
"dag": {
"tasks": {
"component-with-optional-inputs": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-optional-inputs"
},
"inputs": {
"parameters": {
"optional_input_1": {
"componentInputParameter": "input1"
},
"required_input": {
"componentInputParameter": "input0"
}
}
},
"taskInfo": {
"name": "component-with-optional-inputs"
}
}
}
},
"components": {
"comp-component-with-optional-inputs": {
"executorLabel": "exec-component-with-optional-inputs",
"inputDefinitions": {
"parameters": {
"input0": {
"optional_input_1": {
"parameterType": "STRING"
},
"input1": {
"parameterType": "STRING"
},
"input2": {
"required_input": {
"parameterType": "STRING"
}
}
}
},
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.6"
}
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"
}
"defaultPipelineRoot": "dummy_root",
"deploymentSpec": {
"executors": {
"exec-component-with-optional-inputs": {
"container": {
"args": [
"--arg0",
"{{$.inputs.parameters['required_input']}}",
"--arg1",
"{{$.inputs.parameters['optional_input_1']}}",
"--arg3",
"default value"
],
"image": "gcr.io/my-project/my-image"
}
}
}
},
"pipelineInfo": {
"name": "one-step-pipeline-with-if-placeholder"
},
"root": {
"dag": {
"tasks": {
"component-with-optional-inputs": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-optional-inputs"
},
"inputs": {
"parameters": {
"optional_input_1": {
"componentInputParameter": "input1"
},
"required_input": {
"componentInputParameter": "input0"
}
}
},
"taskInfo": {
"name": "component-with-optional-inputs"
}
}
}
},
"inputDefinitions": {
"parameters": {
"input0": {
"parameterType": "STRING"
},
"input1": {
"parameterType": "STRING"
},
"input2": {
"parameterType": "STRING"
}
}
}
},
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.9"
}

5
sdk/python/kfp/v2/components/structures.py
View File

@ -373,8 +373,8 @@ class ComponentSpec(BaseModel):
if arg.if_structure.input_name not in valid_inputs:
raise ValueError(
f'Argument "{arg}" references non-existing input.')
for placeholder in itertools.chain(arg.if_structure.then,
arg.if_structure.otherwise):
for placeholder in itertools.chain(arg.if_structure.then or [],
arg.if_structure.otherwise or []):
cls._check_valid_placeholder_reference(valid_inputs,
valid_outputs,
placeholder)
@ -573,7 +573,6 @@ class ComponentSpec(BaseModel):
Returns:
Component spec in the form of V2 ComponentSpec.
"""
json_component = yaml.safe_load(component_yaml)
try:
return ComponentSpec.parse_obj(json_component)

2
sdk/python/kfp/v2/components/yaml_component.py
View File

@ -43,7 +43,7 @@ def load_component_from_file(file_path: str) -> base_component.BaseComponent:
file_path: A string containing path to the YAML file.
"""
with open(file_path, 'rb') as component_stream:
return load_component_from_text(component_stream)
return load_component_from_text(component_stream.read())
def load_component_from_url(url: str,