feat(sdk)!: use yaml for IR package (#7431)

* change serialization format from json to yaml

* refactor: use guard clause for clarity

* convert test data from json to yaml

* add and update tests

* permit deprecated compile to json

* re-add json test data

* update compiler tests

* refactor Compiler._write_pipeline_spec_file

* update golden snapshots

* update sample output extensions

* test file extension exceptions and warnings

* update release notes

* remove json golden snapshots

samples/test/lightweight_python_functions_v2_pipeline.py

@@ -135,5 +135,5 @@ def pipeline(message: str = 'message'):
 
 if __name__ == '__main__':
     compiler.Compiler().compile(
-        pipeline_func=pipeline, package_path=__file__.replace('.py', '.json')
+        pipeline_func=pipeline, package_path=__file__.replace('.py', '.yaml')
     )

samples/test/lightweight_python_functions_v2_with_outputs.py

@@ -83,4 +83,4 @@ def pipeline(
 
 if __name__ == '__main__':
     compiler.Compiler().compile(
-        pipeline_func=pipeline, package_path=__file__ + '.json')
+        pipeline_func=pipeline, package_path=__file__ + '.yaml')

sdk/RELEASE.md

@@ -13,6 +13,7 @@
 ## Bug Fixes and Other Changes
 
 * Fix wrong kfp import causes wrong sdk_version being set in pipeline_spec. [\#7433](https://github.com/kubeflow/pipelines/pull/7433)
+* Use YAML as default serialization format for package IR [\#7431](https://github.com/kubeflow/pipelines/pull/7431)
 
 ## Documentation Updates
 

sdk/python/kfp/compiler/compiler.py

@@ -17,15 +17,17 @@ This is an experimental implementation of KFP compiler that compiles KFP
 pipeline into Pipeline IR:
 https://docs.google.com/document/d/1PUDuSQ8vmeKSBloli53mp7GIvzekaY7sggg6ywy35Dk/
 """
-
 import collections
 import inspect
+import json
 import re
 import uuid
+import warnings
 from typing import (Any, Callable, Dict, List, Mapping, Optional, Set, Tuple,
                     Union)
 
 import kfp
+import yaml
 from google.protobuf import json_format
 from kfp import dsl
 from kfp.compiler import pipeline_spec_builder as builder
@@ -78,8 +80,8 @@ class Compiler:
 
         Args:
             pipeline_func: Pipeline function with @dsl.pipeline decorator.
-            package_path: The output pipeline spec .json file path. For example,
-                "~/pipeline_spec.json".
+            package_path: The output pipeline spec .yaml file path. For example,
+                "~/pipeline_spec.yaml".
             pipeline_name: Optional; the name of the pipeline.
             pipeline_parameters: Optional; the mapping from parameter names to
                 values.
@@ -94,10 +96,8 @@ class Compiler:
                 pipeline_name=pipeline_name,
                 pipeline_parameters_override=pipeline_parameters,
             )
-            self._write_pipeline_spec_json(
-                pipeline_spec=pipeline_spec,
-                output_path=package_path,
-            )
+            self._write_pipeline_spec_file(
+                pipeline_spec=pipeline_spec, package_path=package_path)
         finally:
             kfp.TYPE_CHECK = type_check_old_value
 
@@ -189,30 +189,39 @@ class Compiler:
 
         return pipeline_spec
 
-    def _write_pipeline_spec_json(
+    def _write_pipeline_spec_file(
         self,
         pipeline_spec: pipeline_spec_pb2.PipelineSpec,
-        output_path: str,
+        package_path: str,
     ) -> None:
-        """Writes pipeline spec into a json file.
-
+        """Writes pipeline spec into a YAML or JSON (deprecated) file.
         Args:
             pipeline_spec: IR pipeline spec.
-            ouput_path: The file path to be written.
-
-        Raises:
-            ValueError: if the specified output path doesn't end with the
-                acceptable extention.
+            package_path: The file path to be written.
         """
-        json_text = json_format.MessageToJson(pipeline_spec, sort_keys=True)
 
-        if output_path.endswith('.json'):
-            with open(output_path, 'w') as json_file:
+        # sort_keys=False retains PipelineSpec's order
+        json_text = json_format.MessageToJson(pipeline_spec, sort_keys=False)
+
+        if package_path.endswith(".json"):
+            warnings.warn(
+                ("Compiling pipline spec to JSON is deprecated and will be "
+                 "removed in a future version. Please compile to a YAML file by "
+                 "providing a file path with .yaml extension instead."),
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            with open(package_path, 'w') as json_file:
                 json_file.write(json_text)
+
+        elif package_path.endswith((".yaml", ".yml")):
+            json_dict = json.loads(json_text)
+            with open(package_path, 'w') as yaml_file:
+                # sort_keys=False retains PipelineSpec's order
+                yaml.dump(json_dict, yaml_file, sort_keys=False)
         else:
             raise ValueError(
-                'The output path {} should ends with ".json".'.format(
-                    output_path))
+                f'The output path {package_path} should end with ".yaml".')
 
     def _validate_exit_handler(self,
                                pipeline: pipeline_context.Pipeline) -> None:

sdk/python/kfp/compiler/compiler_test.py

@@ -15,10 +15,10 @@
 import json
 import os
 import shutil
-import sys
 import tempfile
 import unittest
 
+import yaml
 from absl.testing import parameterized
 from kfp import compiler
 from kfp import components
@@ -85,13 +85,14 @@ class CompilerTest(parameterized.TestCase):
                     input_model=producer.outputs['output_model'],
                     input_value=producer.outputs['output_value'])
 
-            target_json_file = os.path.join(tmpdir, 'result.json')
-            compiler.Compiler().compile(
-                pipeline_func=simple_pipeline, package_path=target_json_file)
+            target_file = os.path.join(tmpdir, 'result.yaml')
 
-            self.assertTrue(os.path.exists(target_json_file))
-            with open(target_json_file, 'r') as f:
-                print(f.read())
+            compiler.Compiler().compile(
+                pipeline_func=simple_pipeline, package_path=target_file)
+
+            self.assertTrue(os.path.exists(target_file))
+            with open(target_file, 'r') as f:
+                f.read()
         finally:
             shutil.rmtree(tmpdir)
 
@@ -114,7 +115,7 @@ class CompilerTest(parameterized.TestCase):
             def simple_pipeline():
                 predict_op(generate_explanation=True)
 
-            target_json_file = os.path.join(tmpdir, 'result.json')
+            target_json_file = os.path.join(tmpdir, 'result.yaml')
             compiler.Compiler().compile(
                 pipeline_func=simple_pipeline, package_path=target_json_file)
 
@@ -167,7 +168,7 @@ class CompilerTest(parameterized.TestCase):
                 TypeError,
                 ' type "Model" cannot be paired with InputValuePlaceholder.'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='output.json')
+                pipeline_func=my_pipeline, package_path='output.yaml')
 
     def test_compile_pipeline_with_misused_inputpath_should_raise_error(self):
 
@@ -190,7 +191,7 @@ class CompilerTest(parameterized.TestCase):
                 TypeError,
                 ' type "String" cannot be paired with InputPathPlaceholder.'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='output.json')
+                pipeline_func=my_pipeline, package_path='output.yaml')
 
     def test_compile_pipeline_with_missing_task_should_raise_error(self):
 
@@ -201,7 +202,7 @@ class CompilerTest(parameterized.TestCase):
         with self.assertRaisesRegex(ValueError,
                                     'Task is missing from pipeline.'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='output.json')
+                pipeline_func=my_pipeline, package_path='output.yaml')
 
     def test_compile_pipeline_with_misused_inputuri_should_raise_error(self):
 
@@ -224,7 +225,7 @@ class CompilerTest(parameterized.TestCase):
                 TypeError,
                 ' type "Float" cannot be paired with InputUriPlaceholder.'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='output.json')
+                pipeline_func=my_pipeline, package_path='output.yaml')
 
     def test_compile_pipeline_with_misused_outputuri_should_raise_error(self):
 
@@ -247,7 +248,7 @@ class CompilerTest(parameterized.TestCase):
                 TypeError,
                 ' type "Integer" cannot be paired with OutputUriPlaceholder.'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='output.json')
+                pipeline_func=my_pipeline, package_path='output.yaml')
 
     def test_compile_pipeline_with_invalid_name_should_raise_error(self):
 
@@ -259,7 +260,7 @@ class CompilerTest(parameterized.TestCase):
                 'Invalid pipeline name: .*\nPlease specify a pipeline name that matches'
         ):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='output.json')
+                pipeline_func=my_pipeline, package_path='output.yaml')
 
     def test_set_pipeline_root_through_pipeline_decorator(self):
 
@@ -270,13 +271,13 @@ class CompilerTest(parameterized.TestCase):
             def my_pipeline():
                 VALID_PRODUCER_COMPONENT_SAMPLE(input_param='input')
 
-            target_json_file = os.path.join(tmpdir, 'result.json')
+            target_json_file = os.path.join(tmpdir, 'result.yaml')
             compiler.Compiler().compile(
                 pipeline_func=my_pipeline, package_path=target_json_file)
 
             self.assertTrue(os.path.exists(target_json_file))
             with open(target_json_file) as f:
-                pipeline_spec = json.load(f)
+                pipeline_spec = yaml.load(f)
             self.assertEqual('gs://path', pipeline_spec['defaultPipelineRoot'])
         finally:
             shutil.rmtree(tmpdir)
@@ -304,7 +305,7 @@ class CompilerTest(parameterized.TestCase):
                 'component "compoent": Argument type "STRING" is incompatible '
                 'with the input type "Artifact"'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='output.json')
+                pipeline_func=my_pipeline, package_path='output.yaml')
 
     def test_passing_missing_type_annotation_on_pipeline_input_should_error(
             self):
@@ -316,7 +317,7 @@ class CompilerTest(parameterized.TestCase):
         with self.assertRaisesRegex(
                 TypeError, 'Missing type annotation for argument: input1'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='output.json')
+                pipeline_func=my_pipeline, package_path='output.yaml')
 
     def test_passing_generic_artifact_to_input_expecting_concrete_artifact(
             self):
@@ -360,11 +361,11 @@ class CompilerTest(parameterized.TestCase):
 
         try:
             tmpdir = tempfile.mkdtemp()
-            target_json_file = os.path.join(tmpdir, 'result.json')
+            target_yaml_file = os.path.join(tmpdir, 'result.yaml')
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path=target_json_file)
+                pipeline_func=my_pipeline, package_path=target_yaml_file)
 
-            self.assertTrue(os.path.exists(target_json_file))
+            self.assertTrue(os.path.exists(target_yaml_file))
         finally:
             shutil.rmtree(tmpdir)
 
@@ -410,11 +411,11 @@ class CompilerTest(parameterized.TestCase):
 
         try:
             tmpdir = tempfile.mkdtemp()
-            target_json_file = os.path.join(tmpdir, 'result.json')
+            target_yaml_file = os.path.join(tmpdir, 'result.yaml')
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path=target_json_file)
+                pipeline_func=my_pipeline, package_path=target_yaml_file)
 
-            self.assertTrue(os.path.exists(target_json_file))
+            self.assertTrue(os.path.exists(target_yaml_file))
         finally:
             shutil.rmtree(tmpdir)
 
@@ -447,7 +448,7 @@ class CompilerTest(parameterized.TestCase):
                 ' "Consumer op": Argument type "SomeArbitraryType" is'
                 ' incompatible with the input type "Dataset"'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='result.json')
+                pipeline_func=my_pipeline, package_path='result.yaml')
 
     @parameterized.parameters(
         {
@@ -504,7 +505,7 @@ class CompilerTest(parameterized.TestCase):
                 RuntimeError,
                 'Task dummy-op cannot dependent on any task inside the group:'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='result.json')
+                pipeline_func=my_pipeline, package_path='result.yaml')
 
     def test_invalid_data_dependency(self):
 
@@ -527,7 +528,7 @@ class CompilerTest(parameterized.TestCase):
                 RuntimeError,
                 'Task dummy-op cannot dependent on any task inside the group:'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='result.json')
+                pipeline_func=my_pipeline, package_path='result.yaml')
 
     def test_use_task_final_status_in_non_exit_op(self):
 
@@ -543,7 +544,7 @@ class CompilerTest(parameterized.TestCase):
                 ValueError,
                 'PipelineTaskFinalStatus can only be used in an exit task.'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='result.json')
+                pipeline_func=my_pipeline, package_path='result.yaml')
 
     def test_use_task_final_status_in_non_exit_op_yaml(self):
 
@@ -567,7 +568,7 @@ implementation:
                 ValueError,
                 'PipelineTaskFinalStatus can only be used in an exit task.'):
             compiler.Compiler().compile(
-                pipeline_func=my_pipeline, package_path='result.json')
+                pipeline_func=my_pipeline, package_path='result.yaml')
 
 
 # pylint: disable=import-outside-toplevel,unused-import,import-error,redefined-outer-name,reimported
@@ -596,12 +597,12 @@ class V2NamespaceAliasTest(unittest.TestCase):
 
         with tempfile.TemporaryDirectory() as tempdir:
             # you can e.g. create a file here:
-            temp_filepath = os.path.join(tempdir, 'hello_world_pipeline.json')
+            temp_filepath = os.path.join(tempdir, 'hello_world_pipeline.yaml')
             v2.compiler.Compiler().compile(
                 pipeline_func=pipeline_hello_world, package_path=temp_filepath)
 
             with open(temp_filepath, "r") as f:
-                json.load(f)
+                yaml.load(f)
 
     def test_import_modules(self):  # pylint: disable=no-self-use
         from kfp.v2 import compiler
@@ -620,12 +621,12 @@ class V2NamespaceAliasTest(unittest.TestCase):
 
         with tempfile.TemporaryDirectory() as tempdir:
             # you can e.g. create a file here:
-            temp_filepath = os.path.join(tempdir, 'hello_world_pipeline.json')
+            temp_filepath = os.path.join(tempdir, 'hello_world_pipeline.yaml')
             compiler.Compiler().compile(
                 pipeline_func=pipeline_hello_world, package_path=temp_filepath)
 
             with open(temp_filepath, "r") as f:
-                json.load(f)
+                yaml.load(f)
 
     def test_import_object(self):  # pylint: disable=no-self-use
         from kfp.v2.compiler import Compiler
@@ -645,12 +646,86 @@ class V2NamespaceAliasTest(unittest.TestCase):
 
         with tempfile.TemporaryDirectory() as tempdir:
             # you can e.g. create a file here:
-            temp_filepath = os.path.join(tempdir, 'hello_world_pipeline.json')
+            temp_filepath = os.path.join(tempdir, 'hello_world_pipeline.yaml')
             Compiler().compile(
                 pipeline_func=pipeline_hello_world, package_path=temp_filepath)
 
             with open(temp_filepath, "r") as f:
-                json.load(f)
+                yaml.load(f)
+
+
+class TestWriteToFileTypes(parameterized.TestCase):
+    pipeline_name = 'test-pipeline'
+
+    def make_pipeline_spec(self):
+
+        @dsl.component
+        def dummy_op():
+            pass
+
+        @dsl.pipeline(name=self.pipeline_name)
+        def my_pipeline():
+            task = dummy_op()
+
+        return my_pipeline
+
+    @parameterized.parameters(
+        {"extension": ".yaml"},
+        {"extension": ".yml"},
+    )
+    def test_can_write_to_yaml(self, extension):
+
+        tmpdir = tempfile.mkdtemp()
+        try:
+            pipeline_spec = self.make_pipeline_spec()
+
+            target_file = os.path.join(tmpdir, f'result{extension}')
+            compiler.Compiler().compile(
+                pipeline_func=pipeline_spec, package_path=target_file)
+
+            self.assertTrue(os.path.exists(target_file))
+            with open(target_file) as f:
+                pipeline_spec = yaml.safe_load(f)
+
+            self.assertEqual(self.pipeline_name,
+                             pipeline_spec['pipelineInfo']['name'])
+
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_can_write_to_json(self):
+
+        tmpdir = tempfile.mkdtemp()
+        try:
+            pipeline_spec = self.make_pipeline_spec()
+
+            target_file = os.path.join(tmpdir, 'result.json')
+            with self.assertWarnsRegex(
+                    DeprecationWarning,
+                    r"Compiling pipline spec to JSON is deprecated"):
+                compiler.Compiler().compile(
+                    pipeline_func=pipeline_spec, package_path=target_file)
+            with open(target_file) as f:
+                pipeline_spec = json.load(f)
+
+            self.assertEqual(self.pipeline_name,
+                             pipeline_spec['pipelineInfo']['name'])
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_cannot_write_to_bad_extension(self):
+
+        tmpdir = tempfile.mkdtemp()
+        try:
+            pipeline_spec = self.make_pipeline_spec()
+
+            target_file = os.path.join(tmpdir, 'result.bad_extension')
+            with self.assertRaisesRegex(ValueError,
+                                        r'.* should end with "\.yaml".*'):
+                compiler.Compiler().compile(
+                    pipeline_func=pipeline_spec, package_path=target_file)
+        finally:
+            shutil.rmtree(tmpdir)
 
 
 if __name__ == '__main__':

sdk/python/kfp/compiler_cli_tests/compiler_cli_tests.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import os
 import re
 import shutil
@@ -20,22 +19,24 @@ import subprocess
 import tempfile
 import unittest
 
+import yaml
+
 
 def _ignore_kfp_version_helper(spec):
     """Ignores kfp sdk versioning in command.
 
-    Takes in a JSON input and ignores the kfp sdk versioning in command
+    Takes in a YAML input and ignores the kfp sdk versioning in command
     for comparison between compiled file and goldens.
     """
     pipeline_spec = spec['pipelineSpec'] if 'pipelineSpec' in spec else spec
 
     if 'executors' in pipeline_spec['deploymentSpec']:
         for executor in pipeline_spec['deploymentSpec']['executors']:
-            pipeline_spec['deploymentSpec']['executors'][executor] = json.loads(
+            pipeline_spec['deploymentSpec']['executors'][executor] = yaml.load(
                 re.sub(
                     "'kfp==(\d+).(\d+).(\d+)(-[a-z]+.\d+)?'", 'kfp',
-                    json.dumps(pipeline_spec['deploymentSpec']['executors']
-                               [executor])))
+                    yaml.dump(pipeline_spec['deploymentSpec']['executors']
+                              [executor])))
     return spec
 
 
@@ -45,7 +46,7 @@ class CompilerCliTests(unittest.TestCase):
         self.maxDiff = None
         return super().setUp()
 
-    def _test_compile_py_to_json(
+    def _test_compile_py_to_yaml(
         self,
         file_base_name,
         additional_arguments=None,
@@ -54,7 +55,7 @@ class CompilerCliTests(unittest.TestCase):
         py_file = os.path.join(test_data_dir, '{}.py'.format(file_base_name))
         tmpdir = tempfile.mkdtemp()
         golden_compiled_file = os.path.join(test_data_dir,
-                                            file_base_name + '.json')
+                                            file_base_name + '.yaml')
 
         if additional_arguments is None:
             additional_arguments = []
@@ -67,7 +68,7 @@ class CompilerCliTests(unittest.TestCase):
 
         def _load_compiled_file(filename: str):
             with open(filename, 'r') as f:
-                contents = json.load(f)
+                contents = yaml.load(f)
                 # Correct the sdkVersion
                 pipeline_spec = contents[
                     'pipelineSpec'] if 'pipelineSpec' in contents else contents
@@ -76,7 +77,7 @@ class CompilerCliTests(unittest.TestCase):
 
         try:
             compiled_file = os.path.join(tmpdir,
-                                         file_base_name + '-pipeline.json')
+                                         file_base_name + '-pipeline.yaml')
             _compile(target_output_file=compiled_file)
 
             golden = _load_compiled_file(golden_compiled_file)
@@ -96,89 +97,89 @@ class CompilerCliTests(unittest.TestCase):
             shutil.rmtree(tmpdir)
 
     def test_two_step_pipeline(self):
-        self._test_compile_py_to_json(
+        self._test_compile_py_to_yaml(
             'two_step_pipeline',
             ['--pipeline-parameters', '{"text":"Hello KFP!"}'])
 
     def test_pipeline_with_importer(self):
-        self._test_compile_py_to_json('pipeline_with_importer')
+        self._test_compile_py_to_yaml('pipeline_with_importer')
 
     def test_pipeline_with_ontology(self):
-        self._test_compile_py_to_json('pipeline_with_ontology')
+        self._test_compile_py_to_yaml('pipeline_with_ontology')
 
     def test_pipeline_with_if_placeholder(self):
-        self._test_compile_py_to_json('pipeline_with_if_placeholder')
+        self._test_compile_py_to_yaml('pipeline_with_if_placeholder')
 
     def test_pipeline_with_concat_placeholder(self):
-        self._test_compile_py_to_json('pipeline_with_concat_placeholder')
+        self._test_compile_py_to_yaml('pipeline_with_concat_placeholder')
 
     def test_pipeline_with_resource_spec(self):
-        self._test_compile_py_to_json('pipeline_with_resource_spec')
+        self._test_compile_py_to_yaml('pipeline_with_resource_spec')
 
     def test_pipeline_with_various_io_types(self):
-        self._test_compile_py_to_json('pipeline_with_various_io_types')
+        self._test_compile_py_to_yaml('pipeline_with_various_io_types')
 
     def test_pipeline_with_reused_component(self):
-        self._test_compile_py_to_json('pipeline_with_reused_component')
+        self._test_compile_py_to_yaml('pipeline_with_reused_component')
 
     def test_pipeline_with_after(self):
-        self._test_compile_py_to_json('pipeline_with_after')
+        self._test_compile_py_to_yaml('pipeline_with_after')
 
     def test_pipeline_with_condition(self):
-        self._test_compile_py_to_json('pipeline_with_condition')
+        self._test_compile_py_to_yaml('pipeline_with_condition')
 
     def test_pipeline_with_nested_conditions(self):
-        self._test_compile_py_to_json('pipeline_with_nested_conditions')
+        self._test_compile_py_to_yaml('pipeline_with_nested_conditions')
 
     def test_pipeline_with_nested_conditions_yaml(self):
-        self._test_compile_py_to_json('pipeline_with_nested_conditions_yaml')
+        self._test_compile_py_to_yaml('pipeline_with_nested_conditions_yaml')
 
     def test_pipeline_with_loops(self):
-        self._test_compile_py_to_json('pipeline_with_loops')
+        self._test_compile_py_to_yaml('pipeline_with_loops')
 
     def test_pipeline_with_nested_loops(self):
-        self._test_compile_py_to_json('pipeline_with_nested_loops')
+        self._test_compile_py_to_yaml('pipeline_with_nested_loops')
 
     def test_pipeline_with_loops_and_conditions(self):
-        self._test_compile_py_to_json('pipeline_with_loops_and_conditions')
+        self._test_compile_py_to_yaml('pipeline_with_loops_and_conditions')
 
     def test_pipeline_with_params_containing_format(self):
-        self._test_compile_py_to_json('pipeline_with_params_containing_format')
+        self._test_compile_py_to_yaml('pipeline_with_params_containing_format')
 
     def test_lightweight_python_functions_v2_pipeline(self):
-        self._test_compile_py_to_json(
+        self._test_compile_py_to_yaml(
             'lightweight_python_functions_v2_pipeline')
 
     def test_lightweight_python_functions_v2_with_outputs(self):
-        self._test_compile_py_to_json(
+        self._test_compile_py_to_yaml(
             'lightweight_python_functions_v2_with_outputs')
 
     def test_xgboost_sample_pipeline(self):
-        self._test_compile_py_to_json('xgboost_sample_pipeline')
+        self._test_compile_py_to_yaml('xgboost_sample_pipeline')
 
     def test_pipeline_with_metrics_outputs(self):
-        self._test_compile_py_to_json('pipeline_with_metrics_outputs')
+        self._test_compile_py_to_yaml('pipeline_with_metrics_outputs')
 
     def test_pipeline_with_exit_handler(self):
-        self._test_compile_py_to_json('pipeline_with_exit_handler')
+        self._test_compile_py_to_yaml('pipeline_with_exit_handler')
 
     def test_pipeline_with_env(self):
-        self._test_compile_py_to_json('pipeline_with_env')
+        self._test_compile_py_to_yaml('pipeline_with_env')
 
     def test_v2_component_with_optional_inputs(self):
-        self._test_compile_py_to_json('v2_component_with_optional_inputs')
+        self._test_compile_py_to_yaml('v2_component_with_optional_inputs')
 
     def test_pipeline_with_gcpc_types(self):
-        self._test_compile_py_to_json('pipeline_with_gcpc_types')
+        self._test_compile_py_to_yaml('pipeline_with_gcpc_types')
 
     def test_pipeline_with_placeholders(self):
-        self._test_compile_py_to_json('pipeline_with_placeholders')
+        self._test_compile_py_to_yaml('pipeline_with_placeholders')
 
     def test_pipeline_with_task_final_status(self):
-        self._test_compile_py_to_json('pipeline_with_task_final_status')
+        self._test_compile_py_to_yaml('pipeline_with_task_final_status')
 
     def test_pipeline_with_task_final_status_yaml(self):
-        self._test_compile_py_to_json('pipeline_with_task_final_status_yaml')
+        self._test_compile_py_to_yaml('pipeline_with_task_final_status_yaml')
 
 
 if __name__ == '__main__':

sdk/python/kfp/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json

@@ -1,249 +0,0 @@
-{
-  "components": {
-    "comp-preprocess": {
-      "executorLabel": "exec-preprocess",
-      "inputDefinitions": {
-        "parameters": {
-          "input_dict_parameter": {
-            "parameterType": "STRUCT"
-          },
-          "input_list_parameter": {
-            "parameterType": "LIST"
-          },
-          "message": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "output_dataset_one": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "output_dataset_two_path": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        },
-        "parameters": {
-          "output_bool_parameter_path": {
-            "parameterType": "BOOLEAN"
-          },
-          "output_dict_parameter_path": {
-            "parameterType": "STRUCT"
-          },
-          "output_list_parameter_path": {
-            "parameterType": "LIST"
-          },
-          "output_parameter_path": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-train": {
-      "executorLabel": "exec-train",
-      "inputDefinitions": {
-        "artifacts": {
-          "dataset_one_path": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "dataset_two": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        },
-        "parameters": {
-          "input_bool": {
-            "parameterType": "BOOLEAN"
-          },
-          "input_dict": {
-            "parameterType": "STRUCT"
-          },
-          "input_list": {
-            "parameterType": "LIST"
-          },
-          "message": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "model": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-preprocess": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "preprocess"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef preprocess(\n    # An input parameter of type string.\n    message: str,\n    # An input parameter of type dict.\n    input_dict_parameter: Dict[str, int],\n    # An input parameter of type list.\n    input_list_parameter: List[str],\n    # Use Output[T] to get a metadata-rich handle to the output artifact\n    # of type `Dataset`.\n    output_dataset_one: Output[Dataset],\n    # A locally accessible filepath for another output artifact of type\n    # `Dataset`.\n    output_dataset_two_path: OutputPath('Dataset'),\n    # A locally accessible filepath for an output parameter of type string.\n    output_parameter_path: OutputPath(str),\n    # A locally accessible filepath for an output parameter of type bool.\n    output_bool_parameter_path: OutputPath(bool),\n    # A locally accessible filepath for an output parameter of type dict.\n    output_dict_parameter_path: OutputPath(Dict[str, int]),\n    # A locally accessible filepath for an output parameter of type list.\n    output_list_parameter_path: OutputPath(List[str]),\n):\n    \"\"\"Dummy preprocessing step.\"\"\"\n\n    # Use Dataset.path to access a local file path for writing.\n    # One can also use Dataset.uri to access the actual URI file path.\n    with open(output_dataset_one.path, 'w') as f:\n        f.write(message)\n\n    # OutputPath is used to just pass the local file path of the output artifact\n    # to the function.\n    with open(output_dataset_two_path, 'w') as f:\n        f.write(message)\n\n    with open(output_parameter_path, 'w') as f:\n        f.write(message)\n\n    with open(output_bool_parameter_path, 'w') as f:\n        f.write(\n            str(True))  # use either `str()` or `json.dumps()` for bool values.\n\n    import json\n    with open(output_dict_parameter_path, 'w') as f:\n        f.write(json.dumps(input_dict_parameter))\n\n    with open(output_list_parameter_path, 'w') as f:\n        f.write(json.dumps(input_list_parameter))\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-train": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "train"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef train(\n    # Use InputPath to get a locally accessible path for the input artifact\n    # of type `Dataset`.\n    dataset_one_path: InputPath('Dataset'),\n    # Use Input[T] to get a metadata-rich handle to the input artifact\n    # of type `Dataset`.\n    dataset_two: Input[Dataset],\n    # An input parameter of type string.\n    message: str,\n    # Use Output[T] to get a metadata-rich handle to the output artifact\n    # of type `Dataset`.\n    model: Output[Model],\n    # An input parameter of type bool.\n    input_bool: bool,\n    # An input parameter of type dict.\n    input_dict: Dict[str, int],\n    # An input parameter of type List[str].\n    input_list: List[str],\n    # An input parameter of type int with a default value.\n    num_steps: int = 100,\n):\n    \"\"\"Dummy Training step.\"\"\"\n    with open(dataset_one_path, 'r') as input_file:\n        dataset_one_contents = input_file.read()\n\n    with open(dataset_two.path, 'r') as input_file:\n        dataset_two_contents = input_file.read()\n\n    line = (f'dataset_one_contents: {dataset_one_contents} || '\n            f'dataset_two_contents: {dataset_two_contents} || '\n            f'message: {message} || '\n            f'input_bool: {input_bool}, type {type(input_bool)} || '\n            f'input_dict: {input_dict}, type {type(input_dict)} || '\n            f'input_list: {input_list}, type {type(input_list)} \\n')\n\n    with open(model.path, 'w') as output_file:\n        for i in range(num_steps):\n            output_file.write('Step {}\\n{}\\n=====\\n'.format(i, line))\n\n    # model is an instance of Model artifact, which has a .metadata dictionary\n    # to store arbitrary metadata for the output artifact.\n    model.metadata['accuracy'] = 0.9\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "my-test-pipeline-beta"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "preprocess": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-preprocess"
-          },
-          "inputs": {
-            "parameters": {
-              "input_dict_parameter": {
-                "componentInputParameter": "input_dict"
-              },
-              "input_list_parameter": {
-                "runtimeValue": {
-                  "constant": [
-                    "a",
-                    "b",
-                    "c"
-                  ]
-                }
-              },
-              "message": {
-                "componentInputParameter": "message"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "preprocess"
-          }
-        },
-        "train": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-train"
-          },
-          "dependentTasks": [
-            "preprocess"
-          ],
-          "inputs": {
-            "artifacts": {
-              "dataset_one_path": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_dataset_one",
-                  "producerTask": "preprocess"
-                }
-              },
-              "dataset_two": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_dataset_two_path",
-                  "producerTask": "preprocess"
-                }
-              }
-            },
-            "parameters": {
-              "input_bool": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "output_bool_parameter_path",
-                  "producerTask": "preprocess"
-                }
-              },
-              "input_dict": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "output_dict_parameter_path",
-                  "producerTask": "preprocess"
-                }
-              },
-              "input_list": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "output_list_parameter_path",
-                  "producerTask": "preprocess"
-                }
-              },
-              "message": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "output_parameter_path",
-                  "producerTask": "preprocess"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "train"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "input_dict": {
-          "defaultValue": {
-            "A": 1.0,
-            "B": 2.0
-          },
-          "parameterType": "STRUCT"
-        },
-        "message": {
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.py

@@ -140,4 +140,4 @@ def pipeline(message: str, input_dict: Dict[str, int] = {'A': 1, 'B': 2}):
 
 if __name__ == '__main__':
     compiler.Compiler().compile(
-        pipeline_func=pipeline, package_path=__file__.replace('.py', '.json'))
+        pipeline_func=pipeline, package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.yaml

@@ -0,0 +1,233 @@
+pipelineInfo:
+  name: my-test-pipeline-beta
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-preprocess:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef preprocess(\n    # An input parameter of type string.\n    message:\
+          \ str,\n    # An input parameter of type dict.\n    input_dict_parameter:\
+          \ Dict[str, int],\n    # An input parameter of type list.\n    input_list_parameter:\
+          \ List[str],\n    # Use Output[T] to get a metadata-rich handle to the output\
+          \ artifact\n    # of type `Dataset`.\n    output_dataset_one: Output[Dataset],\n\
+          \    # A locally accessible filepath for another output artifact of type\n\
+          \    # `Dataset`.\n    output_dataset_two_path: OutputPath('Dataset'),\n\
+          \    # A locally accessible filepath for an output parameter of type string.\n\
+          \    output_parameter_path: OutputPath(str),\n    # A locally accessible\
+          \ filepath for an output parameter of type bool.\n    output_bool_parameter_path:\
+          \ OutputPath(bool),\n    # A locally accessible filepath for an output parameter\
+          \ of type dict.\n    output_dict_parameter_path: OutputPath(Dict[str, int]),\n\
+          \    # A locally accessible filepath for an output parameter of type list.\n\
+          \    output_list_parameter_path: OutputPath(List[str]),\n):\n    \"\"\"\
+          Dummy preprocessing step.\"\"\"\n\n    # Use Dataset.path to access a local\
+          \ file path for writing.\n    # One can also use Dataset.uri to access the\
+          \ actual URI file path.\n    with open(output_dataset_one.path, 'w') as\
+          \ f:\n        f.write(message)\n\n    # OutputPath is used to just pass\
+          \ the local file path of the output artifact\n    # to the function.\n \
+          \   with open(output_dataset_two_path, 'w') as f:\n        f.write(message)\n\
+          \n    with open(output_parameter_path, 'w') as f:\n        f.write(message)\n\
+          \n    with open(output_bool_parameter_path, 'w') as f:\n        f.write(\n\
+          \            str(True))  # use either `str()` or `json.dumps()` for bool\
+          \ values.\n\n    import json\n    with open(output_dict_parameter_path,\
+          \ 'w') as f:\n        f.write(json.dumps(input_dict_parameter))\n\n    with\
+          \ open(output_list_parameter_path, 'w') as f:\n        f.write(json.dumps(input_list_parameter))\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - preprocess
+    exec-train:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef train(\n    # Use InputPath to get a locally accessible path\
+          \ for the input artifact\n    # of type `Dataset`.\n    dataset_one_path:\
+          \ InputPath('Dataset'),\n    # Use Input[T] to get a metadata-rich handle\
+          \ to the input artifact\n    # of type `Dataset`.\n    dataset_two: Input[Dataset],\n\
+          \    # An input parameter of type string.\n    message: str,\n    # Use\
+          \ Output[T] to get a metadata-rich handle to the output artifact\n    #\
+          \ of type `Dataset`.\n    model: Output[Model],\n    # An input parameter\
+          \ of type bool.\n    input_bool: bool,\n    # An input parameter of type\
+          \ dict.\n    input_dict: Dict[str, int],\n    # An input parameter of type\
+          \ List[str].\n    input_list: List[str],\n    # An input parameter of type\
+          \ int with a default value.\n    num_steps: int = 100,\n):\n    \"\"\"Dummy\
+          \ Training step.\"\"\"\n    with open(dataset_one_path, 'r') as input_file:\n\
+          \        dataset_one_contents = input_file.read()\n\n    with open(dataset_two.path,\
+          \ 'r') as input_file:\n        dataset_two_contents = input_file.read()\n\
+          \n    line = (f'dataset_one_contents: {dataset_one_contents} || '\n    \
+          \        f'dataset_two_contents: {dataset_two_contents} || '\n         \
+          \   f'message: {message} || '\n            f'input_bool: {input_bool}, type\
+          \ {type(input_bool)} || '\n            f'input_dict: {input_dict}, type\
+          \ {type(input_dict)} || '\n            f'input_list: {input_list}, type\
+          \ {type(input_list)} \\n')\n\n    with open(model.path, 'w') as output_file:\n\
+          \        for i in range(num_steps):\n            output_file.write('Step\
+          \ {}\\n{}\\n=====\\n'.format(i, line))\n\n    # model is an instance of\
+          \ Model artifact, which has a .metadata dictionary\n    # to store arbitrary\
+          \ metadata for the output artifact.\n    model.metadata['accuracy'] = 0.9\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - train
+components:
+  comp-preprocess:
+    inputDefinitions:
+      parameters:
+        message:
+          parameterType: STRING
+        input_dict_parameter:
+          parameterType: STRUCT
+        input_list_parameter:
+          parameterType: LIST
+    outputDefinitions:
+      artifacts:
+        output_dataset_one:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+        output_dataset_two_path:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+      parameters:
+        output_parameter_path:
+          parameterType: STRING
+        output_bool_parameter_path:
+          parameterType: BOOLEAN
+        output_dict_parameter_path:
+          parameterType: STRUCT
+        output_list_parameter_path:
+          parameterType: LIST
+    executorLabel: exec-preprocess
+  comp-train:
+    inputDefinitions:
+      artifacts:
+        dataset_one_path:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+        dataset_two:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+      parameters:
+        message:
+          parameterType: STRING
+        input_bool:
+          parameterType: BOOLEAN
+        input_dict:
+          parameterType: STRUCT
+        input_list:
+          parameterType: LIST
+    outputDefinitions:
+      artifacts:
+        model:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+    executorLabel: exec-train
+root:
+  inputDefinitions:
+    parameters:
+      message:
+        parameterType: STRING
+      input_dict:
+        parameterType: STRUCT
+        defaultValue:
+          A: 1.0
+          B: 2.0
+  dag:
+    tasks:
+      preprocess:
+        taskInfo:
+          name: preprocess
+        inputs:
+          parameters:
+            message:
+              componentInputParameter: message
+            input_dict_parameter:
+              componentInputParameter: input_dict
+            input_list_parameter:
+              runtimeValue:
+                constant:
+                - a
+                - b
+                - c
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-preprocess
+      train:
+        taskInfo:
+          name: train
+        inputs:
+          parameters:
+            message:
+              taskOutputParameter:
+                producerTask: preprocess
+                outputParameterKey: output_parameter_path
+            input_bool:
+              taskOutputParameter:
+                producerTask: preprocess
+                outputParameterKey: output_bool_parameter_path
+            input_dict:
+              taskOutputParameter:
+                producerTask: preprocess
+                outputParameterKey: output_dict_parameter_path
+            input_list:
+              taskOutputParameter:
+                producerTask: preprocess
+                outputParameterKey: output_list_parameter_path
+          artifacts:
+            dataset_one_path:
+              taskOutputArtifact:
+                producerTask: preprocess
+                outputArtifactKey: output_dataset_one
+            dataset_two:
+              taskOutputArtifact:
+                producerTask: preprocess
+                outputArtifactKey: output_dataset_two_path
+        dependentTasks:
+        - preprocess
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-train
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json

@@ -1,332 +0,0 @@
-{
-  "components": {
-    "comp-add-numbers": {
-      "executorLabel": "exec-add-numbers",
-      "inputDefinitions": {
-        "parameters": {
-          "first": {
-            "parameterType": "NUMBER_INTEGER"
-          },
-          "second": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-concat-message": {
-      "executorLabel": "exec-concat-message",
-      "inputDefinitions": {
-        "parameters": {
-          "first": {
-            "parameterType": "STRING"
-          },
-          "second": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-output-artifact": {
-      "executorLabel": "exec-output-artifact",
-      "inputDefinitions": {
-        "parameters": {
-          "message": {
-            "parameterType": "STRING"
-          },
-          "number": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "Output": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    },
-    "comp-output-named-tuple": {
-      "executorLabel": "exec-output-named-tuple",
-      "inputDefinitions": {
-        "artifacts": {
-          "artifact": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "metrics": {
-            "artifactType": {
-              "schemaTitle": "system.Metrics",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "model": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        },
-        "parameters": {
-          "scalar": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-add-numbers": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "add_numbers"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef add_numbers(first: int, second: int) -> int:\n    return first + second\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-concat-message": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "concat_message"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef concat_message(first: str, second: str) -> str:\n    return first + second\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-output-artifact": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "output_artifact"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef output_artifact(number: int, message: str) -> Dataset:\n    result = [message for _ in range(number)]\n    return '\\n'.join(result)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-output-named-tuple": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "output_named_tuple"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef output_named_tuple(\n    artifact: Input[Dataset]\n) -> NamedTuple('Outputs', [\n    ('scalar', str),\n    ('metrics', Metrics),\n    ('model', Model),\n]):\n    scalar = \"123\"\n\n    import json\n    metrics = json.dumps({\n        'metrics': [{\n            'name': 'accuracy',\n            'numberValue': 0.9,\n            'format': \"PERCENTAGE\",\n        }]\n    })\n\n    with open(artifact.path, 'r') as f:\n        artifact_contents = f.read()\n    model = \"Model contents: \" + artifact_contents\n\n    from collections import namedtuple\n    output = namedtuple('Outputs', ['scalar', 'metrics', 'model'])\n    return output(scalar, metrics, model)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "functions-with-outputs"
-  },
-  "root": {
-    "dag": {
-      "outputs": {
-        "artifacts": {
-          "output-named-tuple-metrics": {
-            "artifactSelectors": [
-              {
-                "outputArtifactKey": "metrics",
-                "producerSubtask": "output-named-tuple"
-              }
-            ]
-          }
-        }
-      },
-      "tasks": {
-        "add-numbers": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-add-numbers"
-          },
-          "inputs": {
-            "parameters": {
-              "first": {
-                "componentInputParameter": "first_number"
-              },
-              "second": {
-                "componentInputParameter": "second_number"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "add-numbers"
-          }
-        },
-        "concat-message": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-concat-message"
-          },
-          "inputs": {
-            "parameters": {
-              "first": {
-                "componentInputParameter": "first_message"
-              },
-              "second": {
-                "componentInputParameter": "second_message"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "concat-message"
-          }
-        },
-        "output-artifact": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-output-artifact"
-          },
-          "dependentTasks": [
-            "add-numbers",
-            "concat-message"
-          ],
-          "inputs": {
-            "parameters": {
-              "message": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "Output",
-                  "producerTask": "concat-message"
-                }
-              },
-              "number": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "Output",
-                  "producerTask": "add-numbers"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "output-artifact"
-          }
-        },
-        "output-named-tuple": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-output-named-tuple"
-          },
-          "dependentTasks": [
-            "output-artifact"
-          ],
-          "inputs": {
-            "artifacts": {
-              "artifact": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "Output",
-                  "producerTask": "output-artifact"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "output-named-tuple"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "first_message": {
-          "parameterType": "STRING"
-        },
-        "first_number": {
-          "parameterType": "NUMBER_INTEGER"
-        },
-        "second_message": {
-          "parameterType": "STRING"
-        },
-        "second_number": {
-          "parameterType": "NUMBER_INTEGER"
-        }
-      }
-    },
-    "outputDefinitions": {
-      "artifacts": {
-        "output-named-tuple-metrics": {
-          "artifactType": {
-            "schemaTitle": "system.Metrics",
-            "schemaVersion": "0.0.1"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.py

@@ -76,4 +76,4 @@ def pipeline(first_message: str, second_message: str, first_number: int,
 
 if __name__ == '__main__':
     compiler.Compiler().compile(
-        pipeline_func=pipeline, package_path=__file__.replace('.py', '.json'))
+        pipeline_func=pipeline, package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.yaml

@@ -0,0 +1,268 @@
+pipelineInfo:
+  name: functions-with-outputs
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-concat-message:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef concat_message(first: str, second: str) -> str:\n    return first\
+          \ + second\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - concat_message
+    exec-add-numbers:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef add_numbers(first: int, second: int) -> int:\n    return first\
+          \ + second\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - add_numbers
+    exec-output-artifact:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef output_artifact(number: int, message: str) -> Dataset:\n    result\
+          \ = [message for _ in range(number)]\n    return '\\n'.join(result)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - output_artifact
+    exec-output-named-tuple:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef output_named_tuple(\n    artifact: Input[Dataset]\n) -> NamedTuple('Outputs',\
+          \ [\n    ('scalar', str),\n    ('metrics', Metrics),\n    ('model', Model),\n\
+          ]):\n    scalar = \"123\"\n\n    import json\n    metrics = json.dumps({\n\
+          \        'metrics': [{\n            'name': 'accuracy',\n            'numberValue':\
+          \ 0.9,\n            'format': \"PERCENTAGE\",\n        }]\n    })\n\n  \
+          \  with open(artifact.path, 'r') as f:\n        artifact_contents = f.read()\n\
+          \    model = \"Model contents: \" + artifact_contents\n\n    from collections\
+          \ import namedtuple\n    output = namedtuple('Outputs', ['scalar', 'metrics',\
+          \ 'model'])\n    return output(scalar, metrics, model)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - output_named_tuple
+components:
+  comp-concat-message:
+    inputDefinitions:
+      parameters:
+        first:
+          parameterType: STRING
+        second:
+          parameterType: STRING
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-concat-message
+  comp-add-numbers:
+    inputDefinitions:
+      parameters:
+        first:
+          parameterType: NUMBER_INTEGER
+        second:
+          parameterType: NUMBER_INTEGER
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: NUMBER_INTEGER
+    executorLabel: exec-add-numbers
+  comp-output-artifact:
+    inputDefinitions:
+      parameters:
+        number:
+          parameterType: NUMBER_INTEGER
+        message:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        Output:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+    executorLabel: exec-output-artifact
+  comp-output-named-tuple:
+    inputDefinitions:
+      artifacts:
+        artifact:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+    outputDefinitions:
+      artifacts:
+        metrics:
+          artifactType:
+            schemaTitle: system.Metrics
+            schemaVersion: 0.0.1
+        model:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+      parameters:
+        scalar:
+          parameterType: STRING
+    executorLabel: exec-output-named-tuple
+root:
+  inputDefinitions:
+    parameters:
+      first_message:
+        parameterType: STRING
+      second_message:
+        parameterType: STRING
+      first_number:
+        parameterType: NUMBER_INTEGER
+      second_number:
+        parameterType: NUMBER_INTEGER
+  outputDefinitions:
+    artifacts:
+      output-named-tuple-metrics:
+        artifactType:
+          schemaTitle: system.Metrics
+          schemaVersion: 0.0.1
+  dag:
+    tasks:
+      concat-message:
+        taskInfo:
+          name: concat-message
+        inputs:
+          parameters:
+            first:
+              componentInputParameter: first_message
+            second:
+              componentInputParameter: second_message
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-concat-message
+      add-numbers:
+        taskInfo:
+          name: add-numbers
+        inputs:
+          parameters:
+            first:
+              componentInputParameter: first_number
+            second:
+              componentInputParameter: second_number
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-add-numbers
+      output-artifact:
+        taskInfo:
+          name: output-artifact
+        inputs:
+          parameters:
+            number:
+              taskOutputParameter:
+                producerTask: add-numbers
+                outputParameterKey: Output
+            message:
+              taskOutputParameter:
+                producerTask: concat-message
+                outputParameterKey: Output
+        dependentTasks:
+        - add-numbers
+        - concat-message
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-output-artifact
+      output-named-tuple:
+        taskInfo:
+          name: output-named-tuple
+        inputs:
+          artifacts:
+            artifact:
+              taskOutputArtifact:
+                producerTask: output-artifact
+                outputArtifactKey: Output
+        dependentTasks:
+        - output-artifact
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-output-named-tuple
+    outputs:
+      artifacts:
+        output-named-tuple-metrics:
+          artifactSelectors:
+          - producerSubtask: output-named-tuple
+            outputArtifactKey: metrics
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_after.json

@@ -1,150 +0,0 @@
-{
-  "components": {
-    "comp-print-text": {
-      "executorLabel": "exec-print-text",
-      "inputDefinitions": {
-        "parameters": {
-          "text": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-text-2": {
-      "executorLabel": "exec-print-text-2",
-      "inputDefinitions": {
-        "parameters": {
-          "text": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-text-3": {
-      "executorLabel": "exec-print-text-3",
-      "inputDefinitions": {
-        "parameters": {
-          "text": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-print-text": {
-        "container": {
-          "command": [
-            "sh",
-            "-c",
-            "set -e -x\necho \"$0\"\n",
-            "{{$.inputs.parameters['text']}}"
-          ],
-          "image": "alpine"
-        }
-      },
-      "exec-print-text-2": {
-        "container": {
-          "command": [
-            "sh",
-            "-c",
-            "set -e -x\necho \"$0\"\n",
-            "{{$.inputs.parameters['text']}}"
-          ],
-          "image": "alpine"
-        }
-      },
-      "exec-print-text-3": {
-        "container": {
-          "command": [
-            "sh",
-            "-c",
-            "set -e -x\necho \"$0\"\n",
-            "{{$.inputs.parameters['text']}}"
-          ],
-          "image": "alpine"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-after"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "print-text": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-text"
-          },
-          "inputs": {
-            "parameters": {
-              "text": {
-                "runtimeValue": {
-                  "constant": "1st task"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-text"
-          }
-        },
-        "print-text-2": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-text-2"
-          },
-          "dependentTasks": [
-            "print-text"
-          ],
-          "inputs": {
-            "parameters": {
-              "text": {
-                "runtimeValue": {
-                  "constant": "2nd task"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-text-2"
-          }
-        },
-        "print-text-3": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-text-3"
-          },
-          "dependentTasks": [
-            "print-text",
-            "print-text-2"
-          ],
-          "inputs": {
-            "parameters": {
-              "text": {
-                "runtimeValue": {
-                  "constant": "3rd task"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-text-3"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_after.py

@@ -43,4 +43,4 @@ def my_pipeline():
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_after.yaml

@@ -0,0 +1,106 @@
+pipelineInfo:
+  name: pipeline-with-after
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-print-text:
+      container:
+        image: alpine
+        command:
+        - sh
+        - -c
+        - 'set -e -x
+
+          echo "$0"
+
+          '
+        - '{{$.inputs.parameters[''text'']}}'
+    exec-print-text-2:
+      container:
+        image: alpine
+        command:
+        - sh
+        - -c
+        - 'set -e -x
+
+          echo "$0"
+
+          '
+        - '{{$.inputs.parameters[''text'']}}'
+    exec-print-text-3:
+      container:
+        image: alpine
+        command:
+        - sh
+        - -c
+        - 'set -e -x
+
+          echo "$0"
+
+          '
+        - '{{$.inputs.parameters[''text'']}}'
+components:
+  comp-print-text:
+    inputDefinitions:
+      parameters:
+        text:
+          parameterType: STRING
+    executorLabel: exec-print-text
+  comp-print-text-2:
+    inputDefinitions:
+      parameters:
+        text:
+          parameterType: STRING
+    executorLabel: exec-print-text-2
+  comp-print-text-3:
+    inputDefinitions:
+      parameters:
+        text:
+          parameterType: STRING
+    executorLabel: exec-print-text-3
+root:
+  dag:
+    tasks:
+      print-text:
+        taskInfo:
+          name: print-text
+        inputs:
+          parameters:
+            text:
+              runtimeValue:
+                constant: 1st task
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-text
+      print-text-2:
+        taskInfo:
+          name: print-text-2
+        inputs:
+          parameters:
+            text:
+              runtimeValue:
+                constant: 2nd task
+        dependentTasks:
+        - print-text
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-text-2
+      print-text-3:
+        taskInfo:
+          name: print-text-3
+        inputs:
+          parameters:
+            text:
+              runtimeValue:
+                constant: 3rd task
+        dependentTasks:
+        - print-text
+        - print-text-2
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-text-3
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json

@@ -1,59 +0,0 @@
-{
-  "components": {
-    "comp-component-with-concat-placeholder": {
-      "executorLabel": "exec-component-with-concat-placeholder",
-      "inputDefinitions": {
-        "parameters": {
-          "input_prefix": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-component-with-concat-placeholder": {
-        "container": {
-          "args": [
-            "--arg0",
-            "{{$.inputs.parameters['input_prefix']}}some value"
-          ],
-          "image": "gcr.io/my-project/my-image"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "one-step-pipeline-with-concat-placeholder"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "component-with-concat-placeholder": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-component-with-concat-placeholder"
-          },
-          "inputs": {
-            "parameters": {
-              "input_prefix": {
-                "runtimeValue": {
-                  "constant": "some prefix:"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "component-with-concat-placeholder"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.py

@@ -33,4 +33,4 @@ def my_pipeline():
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.yaml

@@ -0,0 +1,35 @@
+pipelineInfo:
+  name: one-step-pipeline-with-concat-placeholder
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-component-with-concat-placeholder:
+      container:
+        image: gcr.io/my-project/my-image
+        args:
+        - --arg0
+        - '{{$.inputs.parameters[''input_prefix'']}}some value'
+components:
+  comp-component-with-concat-placeholder:
+    inputDefinitions:
+      parameters:
+        input_prefix:
+          parameterType: STRING
+    executorLabel: exec-component-with-concat-placeholder
+root:
+  dag:
+    tasks:
+      component-with-concat-placeholder:
+        taskInfo:
+          name: component-with-concat-placeholder
+        inputs:
+          parameters:
+            input_prefix:
+              runtimeValue:
+                constant: 'some prefix:'
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-component-with-concat-placeholder
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_condition.json

@@ -1,305 +0,0 @@
-{
-  "components": {
-    "comp-condition-1": {
-      "dag": {
-        "tasks": {
-          "flip-coin-op-2": {
-            "cachingOptions": {},
-            "componentRef": {
-              "name": "comp-flip-coin-op-2"
-            },
-            "taskInfo": {
-              "name": "flip-coin-op-2"
-            }
-          },
-          "print-op-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op-2"
-            },
-            "dependentTasks": [
-              "flip-coin-op-2"
-            ],
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "taskOutputParameter": {
-                    "outputParameterKey": "Output",
-                    "producerTask": "flip-coin-op-2"
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op-2"
-            }
-          },
-          "print-op-3": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op-3"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "componentInputParameter": "pipelinechannel--text"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op-3"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--flip-coin-op-Output": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--text": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-flip-coin-op": {
-      "executorLabel": "exec-flip-coin-op",
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-flip-coin-op-2": {
-      "executorLabel": "exec-flip-coin-op-2",
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op": {
-      "executorLabel": "exec-print-op",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-2": {
-      "executorLabel": "exec-print-op-2",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-3": {
-      "executorLabel": "exec-print-op-3",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-flip-coin-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "flip_coin_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0, 1) == 0 else 'tails'\n    return result\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-flip-coin-op-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "flip_coin_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0, 1) == 0 else 'tails'\n    return result\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-3": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "single-condition-pipeline"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "condition-1": {
-          "componentRef": {
-            "name": "comp-condition-1"
-          },
-          "dependentTasks": [
-            "flip-coin-op"
-          ],
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--flip-coin-op-Output": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "Output",
-                  "producerTask": "flip-coin-op"
-                }
-              },
-              "pipelinechannel--text": {
-                "componentInputParameter": "text"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "condition-1"
-          },
-          "triggerPolicy": {
-            "condition": "inputs.parameter_values['pipelinechannel--flip-coin-op-Output'] == 'heads'"
-          }
-        },
-        "flip-coin-op": {
-          "cachingOptions": {},
-          "componentRef": {
-            "name": "comp-flip-coin-op"
-          },
-          "taskInfo": {
-            "name": "flip-coin-op"
-          }
-        },
-        "print-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op"
-          },
-          "dependentTasks": [
-            "flip-coin-op"
-          ],
-          "inputs": {
-            "parameters": {
-              "msg": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "Output",
-                  "producerTask": "flip-coin-op"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "text": {
-          "defaultValue": "condition test",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_condition.py

@@ -46,4 +46,4 @@ def my_pipeline(text: str = 'condition test'):
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_condition.yaml

@@ -0,0 +1,263 @@
+pipelineInfo:
+  name: single-condition-pipeline
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-flip-coin-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads\
+          \ or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0,\
+          \ 1) == 0 else 'tails'\n    return result\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - flip_coin_op
+    exec-print-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-flip-coin-op-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads\
+          \ or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0,\
+          \ 1) == 0 else 'tails'\n    return result\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - flip_coin_op
+    exec-print-op-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-print-op-3:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+components:
+  comp-condition-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--text:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+    dag:
+      tasks:
+        flip-coin-op-2:
+          taskInfo:
+            name: flip-coin-op-2
+          cachingOptions: {}
+          componentRef:
+            name: comp-flip-coin-op-2
+        print-op-2:
+          taskInfo:
+            name: print-op-2
+          inputs:
+            parameters:
+              msg:
+                taskOutputParameter:
+                  producerTask: flip-coin-op-2
+                  outputParameterKey: Output
+          dependentTasks:
+          - flip-coin-op-2
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op-2
+        print-op-3:
+          taskInfo:
+            name: print-op-3
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--text
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op-3
+  comp-flip-coin-op:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-flip-coin-op
+  comp-print-op:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-op
+  comp-flip-coin-op-2:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-flip-coin-op-2
+  comp-print-op-2:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-op-2
+  comp-print-op-3:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-op-3
+root:
+  inputDefinitions:
+    parameters:
+      text:
+        parameterType: STRING
+        defaultValue: condition test
+  dag:
+    tasks:
+      condition-1:
+        taskInfo:
+          name: condition-1
+        inputs:
+          parameters:
+            pipelinechannel--text:
+              componentInputParameter: text
+            pipelinechannel--flip-coin-op-Output:
+              taskOutputParameter:
+                producerTask: flip-coin-op
+                outputParameterKey: Output
+        dependentTasks:
+        - flip-coin-op
+        componentRef:
+          name: comp-condition-1
+        triggerPolicy:
+          condition: inputs.parameter_values['pipelinechannel--flip-coin-op-Output']
+            == 'heads'
+      flip-coin-op:
+        taskInfo:
+          name: flip-coin-op
+        cachingOptions: {}
+        componentRef:
+          name: comp-flip-coin-op
+      print-op:
+        taskInfo:
+          name: print-op
+        inputs:
+          parameters:
+            msg:
+              taskOutputParameter:
+                producerTask: flip-coin-op
+                outputParameterKey: Output
+        dependentTasks:
+        - flip-coin-op
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_env.json

@@ -1,94 +0,0 @@
-{
-  "components": {
-    "comp-print-env": {
-      "executorLabel": "exec-print-env"
-    },
-    "comp-print-env-op": {
-      "executorLabel": "exec-print-env-op"
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-print-env": {
-        "container": {
-          "command": [
-            "sh",
-            "-c",
-            "set -e -x\necho \"$ENV2\"\necho \"$ENV3\"\n"
-          ],
-          "env": [
-            {
-              "name": "ENV2",
-              "value": "val2"
-            },
-            {
-              "name": "ENV3",
-              "value": "val3"
-            }
-          ],
-          "image": "alpine"
-        }
-      },
-      "exec-print-env-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_env_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_env_op():\n    import os\n    print(os.environ['ENV1'])\n\n"
-          ],
-          "env": [
-            {
-              "name": "ENV1",
-              "value": "val1"
-            }
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-env"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "print-env": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-env"
-          },
-          "taskInfo": {
-            "name": "print-env"
-          }
-        },
-        "print-env-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-env-op"
-          },
-          "taskInfo": {
-            "name": "print-env-op"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_env.py

@@ -52,4 +52,4 @@ def my_pipeline():
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_env.yaml

@@ -0,0 +1,77 @@
+pipelineInfo:
+  name: pipeline-with-env
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-print-env-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_env_op():\n    import os\n    print(os.environ['ENV1'])\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_env_op
+        env:
+        - name: ENV1
+          value: val1
+    exec-print-env:
+      container:
+        image: alpine
+        command:
+        - sh
+        - -c
+        - 'set -e -x
+
+          echo "$ENV2"
+
+          echo "$ENV3"
+
+          '
+        env:
+        - name: ENV2
+          value: val2
+        - name: ENV3
+          value: val3
+components:
+  comp-print-env-op:
+    executorLabel: exec-print-env-op
+  comp-print-env:
+    executorLabel: exec-print-env
+root:
+  dag:
+    tasks:
+      print-env-op:
+        taskInfo:
+          name: print-env-op
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-env-op
+      print-env:
+        taskInfo:
+          name: print-env
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-env
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_exit_handler.json

@@ -1,209 +0,0 @@
-{
-  "components": {
-    "comp-exit-handler-1": {
-      "dag": {
-        "tasks": {
-          "fail-op": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-fail-op"
-            },
-            "inputs": {
-              "parameters": {
-                "message": {
-                  "runtimeValue": {
-                    "constant": "Task failed."
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "fail-op"
-            }
-          },
-          "print-op-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op-2"
-            },
-            "inputs": {
-              "parameters": {
-                "message": {
-                  "componentInputParameter": "pipelinechannel--message"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op-2"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--message": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-fail-op": {
-      "executorLabel": "exec-fail-op",
-      "inputDefinitions": {
-        "parameters": {
-          "message": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op": {
-      "executorLabel": "exec-print-op",
-      "inputDefinitions": {
-        "parameters": {
-          "message": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-2": {
-      "executorLabel": "exec-print-op-2",
-      "inputDefinitions": {
-        "parameters": {
-          "message": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-fail-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "fail_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef fail_op(message: str):\n    \"\"\"Fails.\"\"\"\n    import sys\n    print(message)\n    sys.exit(1)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(message: str):\n    \"\"\"Prints a message.\"\"\"\n    print(message)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(message: str):\n    \"\"\"Prints a message.\"\"\"\n    print(message)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-exit-handler"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "exit-handler-1": {
-          "componentRef": {
-            "name": "comp-exit-handler-1"
-          },
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--message": {
-                "componentInputParameter": "message"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "exit-handler-1"
-          }
-        },
-        "print-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op"
-          },
-          "dependentTasks": [
-            "exit-handler-1"
-          ],
-          "inputs": {
-            "parameters": {
-              "message": {
-                "runtimeValue": {
-                  "constant": "Exit handler has worked!"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op"
-          },
-          "triggerPolicy": {
-            "strategy": "ALL_UPSTREAM_TASKS_COMPLETED"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "message": {
-          "defaultValue": "Hello World!",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_exit_handler.py

@@ -45,4 +45,4 @@ def my_pipeline(message: str = 'Hello World!'):
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_exit_handler.yaml

@@ -0,0 +1,169 @@
+pipelineInfo:
+  name: pipeline-with-exit-handler
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-print-op-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(message: str):\n    \"\"\"Prints a message.\"\"\"\n\
+          \    print(message)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-fail-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef fail_op(message: str):\n    \"\"\"Fails.\"\"\"\n    import sys\n\
+          \    print(message)\n    sys.exit(1)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - fail_op
+    exec-print-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(message: str):\n    \"\"\"Prints a message.\"\"\"\n\
+          \    print(message)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+components:
+  comp-exit-handler-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--message:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-op-2:
+          taskInfo:
+            name: print-op-2
+          inputs:
+            parameters:
+              message:
+                componentInputParameter: pipelinechannel--message
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op-2
+        fail-op:
+          taskInfo:
+            name: fail-op
+          inputs:
+            parameters:
+              message:
+                runtimeValue:
+                  constant: Task failed.
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-fail-op
+  comp-print-op-2:
+    inputDefinitions:
+      parameters:
+        message:
+          parameterType: STRING
+    executorLabel: exec-print-op-2
+  comp-fail-op:
+    inputDefinitions:
+      parameters:
+        message:
+          parameterType: STRING
+    executorLabel: exec-fail-op
+  comp-print-op:
+    inputDefinitions:
+      parameters:
+        message:
+          parameterType: STRING
+    executorLabel: exec-print-op
+root:
+  inputDefinitions:
+    parameters:
+      message:
+        parameterType: STRING
+        defaultValue: Hello World!
+  dag:
+    tasks:
+      exit-handler-1:
+        taskInfo:
+          name: exit-handler-1
+        inputs:
+          parameters:
+            pipelinechannel--message:
+              componentInputParameter: message
+        componentRef:
+          name: comp-exit-handler-1
+      print-op:
+        taskInfo:
+          name: print-op
+        inputs:
+          parameters:
+            message:
+              runtimeValue:
+                constant: Exit handler has worked!
+        dependentTasks:
+        - exit-handler-1
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op
+        triggerPolicy:
+          strategy: ALL_UPSTREAM_TASKS_COMPLETED

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json

@@ -1,111 +0,0 @@
-{
-  "components": {
-    "comp-consumer-op": {
-      "executorLabel": "exec-consumer-op",
-      "inputDefinitions": {
-        "artifacts": {
-          "model": {
-            "artifactType": {
-              "schemaTitle": "google.VertexModel",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    },
-    "comp-producer": {
-      "executorLabel": "exec-producer",
-      "outputDefinitions": {
-        "artifacts": {
-          "model": {
-            "artifactType": {
-              "schemaTitle": "google.VertexModel",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-consumer-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "consumer_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef consumer_op(model: Input[VertexModel]):\n    pass\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-producer": {
-        "container": {
-          "args": [
-            "{{$.outputs.artifacts['model'].path}}"
-          ],
-          "command": [
-            "cmd"
-          ],
-          "image": "dummy"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-gcpc-types"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "consumer-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-consumer-op"
-          },
-          "dependentTasks": [
-            "producer"
-          ],
-          "inputs": {
-            "artifacts": {
-              "model": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "model",
-                  "producerTask": "producer"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "consumer-op"
-          }
-        },
-        "producer": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-producer"
-          },
-          "taskInfo": {
-            "name": "producer"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_gcpc_types.py

@@ -49,4 +49,4 @@ def my_pipeline():
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_gcpc_types.yaml

@@ -0,0 +1,81 @@
+pipelineInfo:
+  name: pipeline-with-gcpc-types
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-producer:
+      container:
+        image: dummy
+        command:
+        - cmd
+        args:
+        - '{{$.outputs.artifacts[''model''].path}}'
+    exec-consumer-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef consumer_op(model: Input[VertexModel]):\n    pass\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - consumer_op
+components:
+  comp-producer:
+    outputDefinitions:
+      artifacts:
+        model:
+          artifactType:
+            schemaTitle: google.VertexModel
+            schemaVersion: 0.0.1
+    executorLabel: exec-producer
+  comp-consumer-op:
+    inputDefinitions:
+      artifacts:
+        model:
+          artifactType:
+            schemaTitle: google.VertexModel
+            schemaVersion: 0.0.1
+    executorLabel: exec-consumer-op
+root:
+  dag:
+    tasks:
+      producer:
+        taskInfo:
+          name: producer
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-producer
+      consumer-op:
+        taskInfo:
+          name: consumer-op
+        inputs:
+          artifacts:
+            model:
+              taskOutputArtifact:
+                producerTask: producer
+                outputArtifactKey: model
+        dependentTasks:
+        - producer
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-consumer-op

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json

@@ -1,80 +0,0 @@
-{
-  "components": {
-    "comp-component-with-optional-inputs": {
-      "executorLabel": "exec-component-with-optional-inputs",
-      "inputDefinitions": {
-        "parameters": {
-          "optional_input_1": {
-            "parameterType": "STRING"
-          },
-          "required_input": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-component-with-optional-inputs": {
-        "container": {
-          "args": [
-            "--arg0",
-            "{{$.inputs.parameters['required_input']}}",
-            "--arg1",
-            "{{$.inputs.parameters['optional_input_1']}}",
-            "--arg3",
-            "default value"
-          ],
-          "image": "gcr.io/my-project/my-image"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "one-step-pipeline-with-if-placeholder"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "component-with-optional-inputs": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-component-with-optional-inputs"
-          },
-          "inputs": {
-            "parameters": {
-              "optional_input_1": {
-                "componentInputParameter": "input1"
-              },
-              "required_input": {
-                "componentInputParameter": "input0"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "component-with-optional-inputs"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "input0": {
-          "parameterType": "STRING"
-        },
-        "input1": {
-          "parameterType": "STRING"
-        },
-        "input2": {
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_if_placeholder.py

@@ -33,4 +33,4 @@ def my_pipeline(input0: str, input1: str, input2: str):
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_if_placeholder.yaml

@@ -0,0 +1,50 @@
+pipelineInfo:
+  name: one-step-pipeline-with-if-placeholder
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-component-with-optional-inputs:
+      container:
+        image: gcr.io/my-project/my-image
+        args:
+        - --arg0
+        - '{{$.inputs.parameters[''required_input'']}}'
+        - --arg1
+        - '{{$.inputs.parameters[''optional_input_1'']}}'
+        - --arg3
+        - default value
+components:
+  comp-component-with-optional-inputs:
+    inputDefinitions:
+      parameters:
+        required_input:
+          parameterType: STRING
+        optional_input_1:
+          parameterType: STRING
+    executorLabel: exec-component-with-optional-inputs
+root:
+  inputDefinitions:
+    parameters:
+      input0:
+        parameterType: STRING
+      input1:
+        parameterType: STRING
+      input2:
+        parameterType: STRING
+  dag:
+    tasks:
+      component-with-optional-inputs:
+        taskInfo:
+          name: component-with-optional-inputs
+        inputs:
+          parameters:
+            required_input:
+              componentInputParameter: input0
+            optional_input_1:
+              componentInputParameter: input1
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-component-with-optional-inputs
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_importer.json

@@ -1,319 +0,0 @@
-{
-  "components": {
-    "comp-condition-1": {
-      "dag": {
-        "tasks": {
-          "importer-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-importer-2"
-            },
-            "inputs": {
-              "parameters": {
-                "uri": {
-                  "componentInputParameter": "pipelinechannel--dataset2"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "importer-2"
-            }
-          },
-          "train-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-train-2"
-            },
-            "dependentTasks": [
-              "importer-2"
-            ],
-            "inputs": {
-              "artifacts": {
-                "dataset": {
-                  "taskOutputArtifact": {
-                    "outputArtifactKey": "artifact",
-                    "producerTask": "importer-2"
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "train-2"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--dataset2": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--train-scalar": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-importer": {
-      "executorLabel": "exec-importer",
-      "inputDefinitions": {
-        "parameters": {
-          "uri": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "artifact": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    },
-    "comp-importer-2": {
-      "executorLabel": "exec-importer-2",
-      "inputDefinitions": {
-        "parameters": {
-          "uri": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "artifact": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    },
-    "comp-train": {
-      "executorLabel": "exec-train",
-      "inputDefinitions": {
-        "artifacts": {
-          "dataset": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "model": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        },
-        "parameters": {
-          "scalar": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-train-2": {
-      "executorLabel": "exec-train-2",
-      "inputDefinitions": {
-        "artifacts": {
-          "dataset": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "model": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        },
-        "parameters": {
-          "scalar": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-importer": {
-        "importer": {
-          "artifactUri": {
-            "constant": "gs://ml-pipeline-playground/shakespeare1.txt"
-          },
-          "metadata": {
-            "key": "value"
-          },
-          "typeSchema": {
-            "schemaTitle": "system.Artifact",
-            "schemaVersion": "0.0.1"
-          }
-        }
-      },
-      "exec-importer-2": {
-        "importer": {
-          "artifactUri": {
-            "runtimeParameter": "uri"
-          },
-          "reimport": true,
-          "typeSchema": {
-            "schemaTitle": "system.Artifact",
-            "schemaVersion": "0.0.1"
-          }
-        }
-      },
-      "exec-train": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "train"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef train(\n    dataset: Input[Dataset]\n) -> NamedTuple('Outputs', [\n    ('scalar', str),\n    ('model', Model),\n]):\n    \"\"\"Dummy Training step.\"\"\"\n    with open(dataset.path, 'r') as f:\n        data = f.read()\n    print('Dataset:', data)\n\n    scalar = '123'\n    model = 'My model trained using data: {}'.format(data)\n\n    from collections import namedtuple\n    output = namedtuple('Outputs', ['scalar', 'model'])\n    return output(scalar, model)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-train-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "train"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef train(\n    dataset: Input[Dataset]\n) -> NamedTuple('Outputs', [\n    ('scalar', str),\n    ('model', Model),\n]):\n    \"\"\"Dummy Training step.\"\"\"\n    with open(dataset.path, 'r') as f:\n        data = f.read()\n    print('Dataset:', data)\n\n    scalar = '123'\n    model = 'My model trained using data: {}'.format(data)\n\n    from collections import namedtuple\n    output = namedtuple('Outputs', ['scalar', 'model'])\n    return output(scalar, model)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-importer"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "condition-1": {
-          "componentRef": {
-            "name": "comp-condition-1"
-          },
-          "dependentTasks": [
-            "train"
-          ],
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--dataset2": {
-                "componentInputParameter": "dataset2"
-              },
-              "pipelinechannel--train-scalar": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "scalar",
-                  "producerTask": "train"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "condition-1"
-          },
-          "triggerPolicy": {
-            "condition": "inputs.parameter_values['pipelinechannel--train-scalar'] == '123'"
-          }
-        },
-        "importer": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-importer"
-          },
-          "inputs": {
-            "parameters": {
-              "uri": {
-                "runtimeValue": {
-                  "constant": "gs://ml-pipeline-playground/shakespeare1.txt"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "importer"
-          }
-        },
-        "train": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-train"
-          },
-          "dependentTasks": [
-            "importer"
-          ],
-          "inputs": {
-            "artifacts": {
-              "dataset": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "artifact",
-                  "producerTask": "importer"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "train"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "dataset2": {
-          "defaultValue": "gs://ml-pipeline-playground/shakespeare2.txt",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_importer.py

@@ -63,4 +63,4 @@ def my_pipeline(dataset2: str = 'gs://ml-pipeline-playground/shakespeare2.txt'):
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_importer.yaml

@@ -0,0 +1,234 @@
+pipelineInfo:
+  name: pipeline-with-importer
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-importer:
+      importer:
+        artifactUri:
+          constant: gs://ml-pipeline-playground/shakespeare1.txt
+        typeSchema:
+          schemaTitle: system.Artifact
+          schemaVersion: 0.0.1
+        metadata:
+          key: value
+    exec-train:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef train(\n    dataset: Input[Dataset]\n) -> NamedTuple('Outputs',\
+          \ [\n    ('scalar', str),\n    ('model', Model),\n]):\n    \"\"\"Dummy Training\
+          \ step.\"\"\"\n    with open(dataset.path, 'r') as f:\n        data = f.read()\n\
+          \    print('Dataset:', data)\n\n    scalar = '123'\n    model = 'My model\
+          \ trained using data: {}'.format(data)\n\n    from collections import namedtuple\n\
+          \    output = namedtuple('Outputs', ['scalar', 'model'])\n    return output(scalar,\
+          \ model)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - train
+    exec-importer-2:
+      importer:
+        artifactUri:
+          runtimeParameter: uri
+        typeSchema:
+          schemaTitle: system.Artifact
+          schemaVersion: 0.0.1
+        reimport: true
+    exec-train-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef train(\n    dataset: Input[Dataset]\n) -> NamedTuple('Outputs',\
+          \ [\n    ('scalar', str),\n    ('model', Model),\n]):\n    \"\"\"Dummy Training\
+          \ step.\"\"\"\n    with open(dataset.path, 'r') as f:\n        data = f.read()\n\
+          \    print('Dataset:', data)\n\n    scalar = '123'\n    model = 'My model\
+          \ trained using data: {}'.format(data)\n\n    from collections import namedtuple\n\
+          \    output = namedtuple('Outputs', ['scalar', 'model'])\n    return output(scalar,\
+          \ model)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - train
+components:
+  comp-condition-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--dataset2:
+          parameterType: STRING
+        pipelinechannel--train-scalar:
+          parameterType: STRING
+    dag:
+      tasks:
+        importer-2:
+          taskInfo:
+            name: importer-2
+          inputs:
+            parameters:
+              uri:
+                componentInputParameter: pipelinechannel--dataset2
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-importer-2
+        train-2:
+          taskInfo:
+            name: train-2
+          inputs:
+            artifacts:
+              dataset:
+                taskOutputArtifact:
+                  producerTask: importer-2
+                  outputArtifactKey: artifact
+          dependentTasks:
+          - importer-2
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-train-2
+  comp-importer:
+    inputDefinitions:
+      parameters:
+        uri:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        artifact:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+    executorLabel: exec-importer
+  comp-train:
+    inputDefinitions:
+      artifacts:
+        dataset:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+    outputDefinitions:
+      artifacts:
+        model:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+      parameters:
+        scalar:
+          parameterType: STRING
+    executorLabel: exec-train
+  comp-importer-2:
+    inputDefinitions:
+      parameters:
+        uri:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        artifact:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+    executorLabel: exec-importer-2
+  comp-train-2:
+    inputDefinitions:
+      artifacts:
+        dataset:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+    outputDefinitions:
+      artifacts:
+        model:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+      parameters:
+        scalar:
+          parameterType: STRING
+    executorLabel: exec-train-2
+root:
+  inputDefinitions:
+    parameters:
+      dataset2:
+        parameterType: STRING
+        defaultValue: gs://ml-pipeline-playground/shakespeare2.txt
+  dag:
+    tasks:
+      condition-1:
+        taskInfo:
+          name: condition-1
+        inputs:
+          parameters:
+            pipelinechannel--dataset2:
+              componentInputParameter: dataset2
+            pipelinechannel--train-scalar:
+              taskOutputParameter:
+                producerTask: train
+                outputParameterKey: scalar
+        dependentTasks:
+        - train
+        componentRef:
+          name: comp-condition-1
+        triggerPolicy:
+          condition: inputs.parameter_values['pipelinechannel--train-scalar'] == '123'
+      importer:
+        taskInfo:
+          name: importer
+        inputs:
+          parameters:
+            uri:
+              runtimeValue:
+                constant: gs://ml-pipeline-playground/shakespeare1.txt
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-importer
+      train:
+        taskInfo:
+          name: train
+        inputs:
+          artifacts:
+            dataset:
+              taskOutputArtifact:
+                producerTask: importer
+                outputArtifactKey: artifact
+        dependentTasks:
+        - importer
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-train
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_loops.json

@@ -1,514 +0,0 @@
-{
-  "components": {
-    "comp-args-generator-op": {
-      "executorLabel": "exec-args-generator-op",
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "LIST"
-          }
-        }
-      }
-    },
-    "comp-for-loop-1": {
-      "dag": {
-        "tasks": {
-          "print-text": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-text"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "componentInputParameter": "pipelinechannel--loop_parameter-loop-item"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-text"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--loop_parameter": {
-            "parameterType": "LIST"
-          },
-          "pipelinechannel--loop_parameter-loop-item": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-for-loop-2": {
-      "dag": {
-        "tasks": {
-          "print-struct": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-struct"
-            },
-            "inputs": {
-              "parameters": {
-                "struct": {
-                  "componentInputParameter": "pipelinechannel--args-generator-op-Output-loop-item"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-struct"
-            }
-          },
-          "print-text-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-text-2"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "componentInputParameter": "pipelinechannel--args-generator-op-Output-loop-item",
-                  "parameterExpressionSelector": "parseJson(string_value)[\"A_a\"]"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-text-2"
-            }
-          },
-          "print-text-3": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-text-3"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "componentInputParameter": "pipelinechannel--args-generator-op-Output-loop-item",
-                  "parameterExpressionSelector": "parseJson(string_value)[\"B_b\"]"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-text-3"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--args-generator-op-Output": {
-            "parameterType": "LIST"
-          },
-          "pipelinechannel--args-generator-op-Output-loop-item": {
-            "parameterType": "STRUCT"
-          }
-        }
-      }
-    },
-    "comp-for-loop-4": {
-      "dag": {
-        "tasks": {
-          "print-struct-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-struct-2"
-            },
-            "inputs": {
-              "parameters": {
-                "struct": {
-                  "componentInputParameter": "pipelinechannel--loop-item-param-3"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-struct-2"
-            }
-          },
-          "print-text-4": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-text-4"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "componentInputParameter": "pipelinechannel--loop-item-param-3",
-                  "parameterExpressionSelector": "parseJson(string_value)[\"A_a\"]"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-text-4"
-            }
-          },
-          "print-text-5": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-text-5"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "componentInputParameter": "pipelinechannel--loop-item-param-3",
-                  "parameterExpressionSelector": "parseJson(string_value)[\"B_b\"]"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-text-5"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--loop-item-param-3": {
-            "parameterType": "STRUCT"
-          }
-        }
-      }
-    },
-    "comp-print-struct": {
-      "executorLabel": "exec-print-struct",
-      "inputDefinitions": {
-        "parameters": {
-          "struct": {
-            "parameterType": "STRUCT"
-          }
-        }
-      }
-    },
-    "comp-print-struct-2": {
-      "executorLabel": "exec-print-struct-2",
-      "inputDefinitions": {
-        "parameters": {
-          "struct": {
-            "parameterType": "STRUCT"
-          }
-        }
-      }
-    },
-    "comp-print-text": {
-      "executorLabel": "exec-print-text",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-text-2": {
-      "executorLabel": "exec-print-text-2",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-text-3": {
-      "executorLabel": "exec-print-text-3",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-text-4": {
-      "executorLabel": "exec-print-text-4",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-text-5": {
-      "executorLabel": "exec-print-text-5",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-args-generator-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "args_generator_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef args_generator_op() -> List[Dict[str, str]]:\n    return [{'A_a': '1', 'B_b': '2'}, {'A_a': '10', 'B_b': '20'}]\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-struct": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_struct"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_struct(struct: Dict):\n    print(struct)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-struct-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_struct"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_struct(struct: Dict):\n    print(struct)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-text": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_text"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-text-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_text"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-text-3": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_text"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-text-4": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_text"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-text-5": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_text"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-loops"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "args-generator-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-args-generator-op"
-          },
-          "taskInfo": {
-            "name": "args-generator-op"
-          }
-        },
-        "for-loop-1": {
-          "componentRef": {
-            "name": "comp-for-loop-1"
-          },
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--loop_parameter": {
-                "componentInputParameter": "loop_parameter"
-              }
-            }
-          },
-          "parameterIterator": {
-            "itemInput": "pipelinechannel--loop_parameter-loop-item",
-            "items": {
-              "inputParameter": "pipelinechannel--loop_parameter"
-            }
-          },
-          "taskInfo": {
-            "name": "for-loop-1"
-          }
-        },
-        "for-loop-2": {
-          "componentRef": {
-            "name": "comp-for-loop-2"
-          },
-          "dependentTasks": [
-            "args-generator-op"
-          ],
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--args-generator-op-Output": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "Output",
-                  "producerTask": "args-generator-op"
-                }
-              }
-            }
-          },
-          "parameterIterator": {
-            "itemInput": "pipelinechannel--args-generator-op-Output-loop-item",
-            "items": {
-              "inputParameter": "pipelinechannel--args-generator-op-Output"
-            }
-          },
-          "taskInfo": {
-            "name": "for-loop-2"
-          }
-        },
-        "for-loop-4": {
-          "componentRef": {
-            "name": "comp-for-loop-4"
-          },
-          "parameterIterator": {
-            "itemInput": "pipelinechannel--loop-item-param-3",
-            "items": {
-              "raw": "[{\"A_a\": \"1\", \"B_b\": \"2\"}, {\"A_a\": \"10\", \"B_b\": \"20\"}]"
-            }
-          },
-          "taskInfo": {
-            "name": "for-loop-4"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "loop_parameter": {
-          "parameterType": "LIST"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_loops.py

@@ -58,4 +58,4 @@ def my_pipeline(loop_parameter: List[str]):
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_loops.yaml

@@ -0,0 +1,423 @@
+pipelineInfo:
+  name: pipeline-with-loops
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-args-generator-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef args_generator_op() -> List[Dict[str, str]]:\n    return [{'A_a':\
+          \ '1', 'B_b': '2'}, {'A_a': '10', 'B_b': '20'}]\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - args_generator_op
+    exec-print-text:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-struct:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_struct(struct: Dict):\n    print(struct)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_struct
+    exec-print-text-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-text-3:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-struct-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_struct(struct: Dict):\n    print(struct)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_struct
+    exec-print-text-4:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-text-5:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str):\n    print(msg)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+components:
+  comp-for-loop-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop_parameter:
+          parameterType: LIST
+        pipelinechannel--loop_parameter-loop-item:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-text:
+          taskInfo:
+            name: print-text
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text
+  comp-for-loop-2:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--args-generator-op-Output:
+          parameterType: LIST
+        pipelinechannel--args-generator-op-Output-loop-item:
+          parameterType: STRUCT
+    dag:
+      tasks:
+        print-struct:
+          taskInfo:
+            name: print-struct
+          inputs:
+            parameters:
+              struct:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-struct
+        print-text-2:
+          taskInfo:
+            name: print-text-2
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+                parameterExpressionSelector: parseJson(string_value)["A_a"]
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-2
+        print-text-3:
+          taskInfo:
+            name: print-text-3
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+                parameterExpressionSelector: parseJson(string_value)["B_b"]
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-3
+  comp-for-loop-4:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop-item-param-3:
+          parameterType: STRUCT
+    dag:
+      tasks:
+        print-struct-2:
+          taskInfo:
+            name: print-struct-2
+          inputs:
+            parameters:
+              struct:
+                componentInputParameter: pipelinechannel--loop-item-param-3
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-struct-2
+        print-text-4:
+          taskInfo:
+            name: print-text-4
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop-item-param-3
+                parameterExpressionSelector: parseJson(string_value)["A_a"]
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-4
+        print-text-5:
+          taskInfo:
+            name: print-text-5
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop-item-param-3
+                parameterExpressionSelector: parseJson(string_value)["B_b"]
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-5
+  comp-args-generator-op:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: LIST
+    executorLabel: exec-args-generator-op
+  comp-print-text:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text
+  comp-print-struct:
+    inputDefinitions:
+      parameters:
+        struct:
+          parameterType: STRUCT
+    executorLabel: exec-print-struct
+  comp-print-text-2:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-2
+  comp-print-text-3:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-3
+  comp-print-struct-2:
+    inputDefinitions:
+      parameters:
+        struct:
+          parameterType: STRUCT
+    executorLabel: exec-print-struct-2
+  comp-print-text-4:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-4
+  comp-print-text-5:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-5
+root:
+  inputDefinitions:
+    parameters:
+      loop_parameter:
+        parameterType: LIST
+  dag:
+    tasks:
+      for-loop-1:
+        taskInfo:
+          name: for-loop-1
+        inputs:
+          parameters:
+            pipelinechannel--loop_parameter:
+              componentInputParameter: loop_parameter
+        componentRef:
+          name: comp-for-loop-1
+        parameterIterator:
+          items:
+            inputParameter: pipelinechannel--loop_parameter
+          itemInput: pipelinechannel--loop_parameter-loop-item
+      for-loop-2:
+        taskInfo:
+          name: for-loop-2
+        inputs:
+          parameters:
+            pipelinechannel--args-generator-op-Output:
+              taskOutputParameter:
+                producerTask: args-generator-op
+                outputParameterKey: Output
+        dependentTasks:
+        - args-generator-op
+        componentRef:
+          name: comp-for-loop-2
+        parameterIterator:
+          items:
+            inputParameter: pipelinechannel--args-generator-op-Output
+          itemInput: pipelinechannel--args-generator-op-Output-loop-item
+      for-loop-4:
+        taskInfo:
+          name: for-loop-4
+        componentRef:
+          name: comp-for-loop-4
+        parameterIterator:
+          items:
+            raw: '[{"A_a": "1", "B_b": "2"}, {"A_a": "10", "B_b": "20"}]'
+          itemInput: pipelinechannel--loop-item-param-3
+      args-generator-op:
+        taskInfo:
+          name: args-generator-op
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-args-generator-op

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.py

@@ -111,4 +111,4 @@ def my_pipeline(
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.yaml

@@ -0,0 +1,993 @@
+pipelineInfo:
+  name: pipeline-with-loops-and-conditions-multi-layers
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-flip-coin-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads\
+          \ or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0,\
+          \ 1) == 0 else 'tails'\n    return result\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - flip_coin_op
+    exec-args-generator-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef args_generator_op() -> list:\n    return [\n        {\n     \
+          \       'A_a': '1',\n            'B_b': ['2', '20'],\n        },\n     \
+          \   {\n            'A_a': '10',\n            'B_b': ['22', '222'],\n   \
+          \     },\n    ]\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - args_generator_op
+    exec-args-generator-op-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef args_generator_op() -> list:\n    return [\n        {\n     \
+          \       'A_a': '1',\n            'B_b': ['2', '20'],\n        },\n     \
+          \   {\n            'A_a': '10',\n            'B_b': ['22', '222'],\n   \
+          \     },\n    ]\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - args_generator_op
+    exec-print-text:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-text-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-text-3:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-struct:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_struct(struct: dict):\n    print(struct)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_struct
+    exec-print-text-4:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-text-5:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-text-6:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-text-7:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-text-8:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+    exec-print-text-9:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_text
+components:
+  comp-condition-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop_parameter:
+          parameterType: LIST
+        pipelinechannel--msg:
+          parameterType: STRING
+        pipelinechannel--args-generator-op-Output:
+          parameterType: LIST
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+    dag:
+      tasks:
+        for-loop-2:
+          taskInfo:
+            name: for-loop-2
+          inputs:
+            parameters:
+              pipelinechannel--loop_parameter:
+                componentInputParameter: pipelinechannel--loop_parameter
+              pipelinechannel--args-generator-op-2-Output:
+                taskOutputParameter:
+                  producerTask: args-generator-op-2
+                  outputParameterKey: Output
+              pipelinechannel--args-generator-op-Output:
+                componentInputParameter: pipelinechannel--args-generator-op-Output
+              pipelinechannel--msg:
+                componentInputParameter: pipelinechannel--msg
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+          dependentTasks:
+          - args-generator-op-2
+          componentRef:
+            name: comp-for-loop-2
+          parameterIterator:
+            items:
+              inputParameter: pipelinechannel--args-generator-op-Output
+            itemInput: pipelinechannel--args-generator-op-Output-loop-item
+        args-generator-op-2:
+          taskInfo:
+            name: args-generator-op-2
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-args-generator-op-2
+  comp-for-loop-14:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop_parameter:
+          parameterType: LIST
+        pipelinechannel--loop_parameter-loop-item:
+          parameterType: STRING
+    dag:
+      tasks:
+        condition-15:
+          taskInfo:
+            name: condition-15
+          inputs:
+            parameters:
+              pipelinechannel--loop_parameter-loop-item:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+              pipelinechannel--loop_parameter-loop-item-subvar-A_a:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+                parameterExpressionSelector: parseJson(string_value)["A_a"]
+          componentRef:
+            name: comp-condition-15
+          triggerPolicy:
+            condition: inputs.parameter_values['pipelinechannel--loop_parameter-loop-item-subvar-A_a']
+              == 'heads'
+  comp-flip-coin-op:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-flip-coin-op
+  comp-args-generator-op:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: LIST
+    executorLabel: exec-args-generator-op
+  comp-for-loop-2:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop_parameter:
+          parameterType: LIST
+        pipelinechannel--args-generator-op-2-Output:
+          parameterType: LIST
+        pipelinechannel--args-generator-op-Output:
+          parameterType: LIST
+        pipelinechannel--msg:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+        pipelinechannel--args-generator-op-Output-loop-item:
+          parameterType: STRING
+    dag:
+      tasks:
+        condition-3:
+          taskInfo:
+            name: condition-3
+          inputs:
+            parameters:
+              pipelinechannel--args-generator-op-Output-loop-item:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+              pipelinechannel--args-generator-op-Output-loop-item-subvar-A_a:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+                parameterExpressionSelector: parseJson(string_value)["A_a"]
+          componentRef:
+            name: comp-condition-3
+          triggerPolicy:
+            condition: inputs.parameter_values['pipelinechannel--args-generator-op-Output-loop-item-subvar-A_a']
+              == 'heads'
+        condition-4:
+          taskInfo:
+            name: condition-4
+          inputs:
+            parameters:
+              pipelinechannel--args-generator-op-Output-loop-item:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+          componentRef:
+            name: comp-condition-4
+          triggerPolicy:
+            condition: inputs.parameter_values['pipelinechannel--flip-coin-op-Output']
+              == 'heads'
+        condition-5:
+          taskInfo:
+            name: condition-5
+          inputs:
+            parameters:
+              pipelinechannel--args-generator-op-Output-loop-item:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+              pipelinechannel--args-generator-op-Output-loop-item-subvar-A_a:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+                parameterExpressionSelector: parseJson(string_value)["A_a"]
+          componentRef:
+            name: comp-condition-5
+          triggerPolicy:
+            condition: inputs.parameter_values['pipelinechannel--args-generator-op-Output-loop-item-subvar-A_a']
+              == 'tails'
+        for-loop-8:
+          taskInfo:
+            name: for-loop-8
+          inputs:
+            parameters:
+              pipelinechannel--args-generator-op-Output-loop-item:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+              pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+                parameterExpressionSelector: parseJson(string_value)["B_b"]
+          componentRef:
+            name: comp-for-loop-8
+          parameterIterator:
+            items:
+              inputParameter: pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b
+            itemInput: pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b-loop-item
+        for-loop-9:
+          taskInfo:
+            name: for-loop-9
+          inputs:
+            parameters:
+              pipelinechannel--loop_parameter:
+                componentInputParameter: pipelinechannel--loop_parameter
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+              pipelinechannel--args-generator-op-2-Output:
+                componentInputParameter: pipelinechannel--args-generator-op-2-Output
+          componentRef:
+            name: comp-for-loop-9
+          parameterIterator:
+            items:
+              inputParameter: pipelinechannel--loop_parameter
+            itemInput: pipelinechannel--loop_parameter-loop-item
+        for-loop-12:
+          taskInfo:
+            name: for-loop-12
+          inputs:
+            parameters:
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+          componentRef:
+            name: comp-for-loop-12
+          parameterIterator:
+            items:
+              raw: '["1", "2"]'
+            itemInput: pipelinechannel--loop-item-param-11
+        print-text:
+          taskInfo:
+            name: print-text
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--msg
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text
+  comp-args-generator-op-2:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: LIST
+    executorLabel: exec-args-generator-op-2
+  comp-condition-3:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--args-generator-op-Output-loop-item:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+        pipelinechannel--args-generator-op-Output-loop-item-subvar-A_a:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-text-2:
+          taskInfo:
+            name: print-text-2
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+                parameterExpressionSelector: parseJson(string_value)["B_b"]
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-2
+  comp-condition-4:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--args-generator-op-Output-loop-item:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-text-3:
+          taskInfo:
+            name: print-text-3
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+                parameterExpressionSelector: parseJson(string_value)["B_b"]
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-3
+  comp-condition-5:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--args-generator-op-Output-loop-item:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+        pipelinechannel--args-generator-op-Output-loop-item-subvar-A_a:
+          parameterType: STRING
+    dag:
+      tasks:
+        for-loop-7:
+          taskInfo:
+            name: for-loop-7
+          inputs:
+            parameters:
+              pipelinechannel--args-generator-op-Output-loop-item:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+          componentRef:
+            name: comp-for-loop-7
+          parameterIterator:
+            items:
+              raw: '[{"a": "-1"}, {"a": "-2"}]'
+            itemInput: pipelinechannel--loop-item-param-6
+  comp-for-loop-8:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--args-generator-op-Output-loop-item:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+        pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b:
+          parameterType: STRING
+        pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b-loop-item:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-text-4:
+          taskInfo:
+            name: print-text-4
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b-loop-item
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-4
+  comp-for-loop-9:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop_parameter:
+          parameterType: LIST
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+        pipelinechannel--args-generator-op-2-Output:
+          parameterType: LIST
+        pipelinechannel--loop_parameter-loop-item:
+          parameterType: STRING
+    dag:
+      tasks:
+        for-loop-10:
+          taskInfo:
+            name: for-loop-10
+          inputs:
+            parameters:
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+              pipelinechannel--args-generator-op-2-Output:
+                componentInputParameter: pipelinechannel--args-generator-op-2-Output
+              pipelinechannel--loop_parameter-loop-item:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+          componentRef:
+            name: comp-for-loop-10
+          parameterIterator:
+            items:
+              inputParameter: pipelinechannel--args-generator-op-2-Output
+            itemInput: pipelinechannel--args-generator-op-2-Output-loop-item
+        print-text-5:
+          taskInfo:
+            name: print-text-5
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-5
+  comp-for-loop-12:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop-item-param-11:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+    dag:
+      tasks:
+        condition-13:
+          taskInfo:
+            name: condition-13
+          inputs:
+            parameters:
+              pipelinechannel--loop-item-param-11:
+                componentInputParameter: pipelinechannel--loop-item-param-11
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+          componentRef:
+            name: comp-condition-13
+          triggerPolicy:
+            condition: inputs.parameter_values['pipelinechannel--loop-item-param-11']
+              == '1'
+        print-text-7:
+          taskInfo:
+            name: print-text-7
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop-item-param-11
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-7
+  comp-print-text:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text
+  comp-print-text-2:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-2
+  comp-print-text-3:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-3
+  comp-for-loop-7:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--args-generator-op-Output-loop-item:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+        pipelinechannel--loop-item-param-6:
+          parameterType: STRUCT
+    dag:
+      tasks:
+        print-struct:
+          taskInfo:
+            name: print-struct
+          inputs:
+            parameters:
+              struct:
+                componentInputParameter: pipelinechannel--loop-item-param-6
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-struct
+  comp-print-struct:
+    inputDefinitions:
+      parameters:
+        struct:
+          parameterType: STRUCT
+    executorLabel: exec-print-struct
+  comp-print-text-4:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-4
+  comp-for-loop-10:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+        pipelinechannel--args-generator-op-2-Output:
+          parameterType: LIST
+        pipelinechannel--loop_parameter-loop-item:
+          parameterType: STRING
+        pipelinechannel--args-generator-op-2-Output-loop-item:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-text-6:
+          taskInfo:
+            name: print-text-6
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+              msg2:
+                componentInputParameter: pipelinechannel--args-generator-op-2-Output-loop-item
+                parameterExpressionSelector: parseJson(string_value)["A_a"]
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-6
+  comp-print-text-5:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-5
+  comp-print-text-6:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+        msg2:
+          parameterType: STRING
+    executorLabel: exec-print-text-6
+  comp-condition-13:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop-item-param-11:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-text-8:
+          taskInfo:
+            name: print-text-8
+          inputs:
+            parameters:
+              msg:
+                runtimeValue:
+                  constant: '1'
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-8
+  comp-print-text-7:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-7
+  comp-print-text-8:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-8
+  comp-condition-15:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop_parameter-loop-item:
+          parameterType: STRING
+        pipelinechannel--loop_parameter-loop-item-subvar-A_a:
+          parameterType: STRING
+    dag:
+      tasks:
+        for-loop-16:
+          taskInfo:
+            name: for-loop-16
+          inputs:
+            parameters:
+              pipelinechannel--loop_parameter-loop-item:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+              pipelinechannel--loop_parameter-loop-item-subvar-B_b:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+                parameterExpressionSelector: parseJson(string_value)["B_b"]
+          componentRef:
+            name: comp-for-loop-16
+          parameterIterator:
+            items:
+              inputParameter: pipelinechannel--loop_parameter-loop-item-subvar-B_b
+            itemInput: pipelinechannel--loop_parameter-loop-item-subvar-B_b-loop-item
+  comp-for-loop-16:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop_parameter-loop-item:
+          parameterType: STRING
+        pipelinechannel--loop_parameter-loop-item-subvar-B_b:
+          parameterType: STRING
+        pipelinechannel--loop_parameter-loop-item-subvar-B_b-loop-item:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-text-9:
+          taskInfo:
+            name: print-text-9
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item-subvar-B_b-loop-item
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-text-9
+  comp-print-text-9:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-text-9
+root:
+  inputDefinitions:
+    parameters:
+      msg:
+        parameterType: STRING
+        defaultValue: hello
+      loop_parameter:
+        parameterType: LIST
+        defaultValue:
+        - A_a: heads
+          B_b:
+          - A
+          - B
+        - A_a: tails
+          B_b:
+          - X
+          - Y
+          - Z
+  dag:
+    tasks:
+      condition-1:
+        taskInfo:
+          name: condition-1
+        inputs:
+          parameters:
+            pipelinechannel--loop_parameter:
+              componentInputParameter: loop_parameter
+            pipelinechannel--msg:
+              componentInputParameter: msg
+            pipelinechannel--args-generator-op-Output:
+              taskOutputParameter:
+                producerTask: args-generator-op
+                outputParameterKey: Output
+            pipelinechannel--flip-coin-op-Output:
+              taskOutputParameter:
+                producerTask: flip-coin-op
+                outputParameterKey: Output
+        dependentTasks:
+        - args-generator-op
+        - flip-coin-op
+        componentRef:
+          name: comp-condition-1
+        triggerPolicy:
+          condition: inputs.parameter_values['pipelinechannel--flip-coin-op-Output']
+            != 'no-such-result'
+      for-loop-14:
+        taskInfo:
+          name: for-loop-14
+        inputs:
+          parameters:
+            pipelinechannel--loop_parameter:
+              componentInputParameter: loop_parameter
+        componentRef:
+          name: comp-for-loop-14
+        parameterIterator:
+          items:
+            inputParameter: pipelinechannel--loop_parameter
+          itemInput: pipelinechannel--loop_parameter-loop-item
+      flip-coin-op:
+        taskInfo:
+          name: flip-coin-op
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-flip-coin-op
+      args-generator-op:
+        taskInfo:
+          name: args-generator-op
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-args-generator-op

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json

@@ -1,193 +0,0 @@
-{
-  "components": {
-    "comp-for-loop-2": {
-      "dag": {
-        "outputs": {
-          "artifacts": {
-            "output-metrics-2-metrics": {
-              "artifactSelectors": [
-                {
-                  "outputArtifactKey": "metrics",
-                  "producerSubtask": "output-metrics-2"
-                }
-              ]
-            }
-          }
-        },
-        "tasks": {
-          "output-metrics-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-output-metrics-2"
-            },
-            "taskInfo": {
-              "name": "output-metrics-2"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--loop-item-param-1": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "output-metrics-2-metrics": {
-            "artifactType": {
-              "schemaTitle": "system.Metrics",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    },
-    "comp-output-metrics": {
-      "executorLabel": "exec-output-metrics",
-      "outputDefinitions": {
-        "artifacts": {
-          "metrics": {
-            "artifactType": {
-              "schemaTitle": "system.Metrics",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    },
-    "comp-output-metrics-2": {
-      "executorLabel": "exec-output-metrics-2",
-      "outputDefinitions": {
-        "artifacts": {
-          "metrics": {
-            "artifactType": {
-              "schemaTitle": "system.Metrics",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-output-metrics": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "output_metrics"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef output_metrics(metrics: Output[Metrics]):\n    \"\"\"Dummy component that outputs metrics with a random accuracy.\"\"\"\n    import random\n    result = random.randint(0, 100)\n    metrics.log_metric('accuracy', result)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-output-metrics-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "output_metrics"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef output_metrics(metrics: Output[Metrics]):\n    \"\"\"Dummy component that outputs metrics with a random accuracy.\"\"\"\n    import random\n    result = random.randint(0, 100)\n    metrics.log_metric('accuracy', result)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-metrics-outputs"
-  },
-  "root": {
-    "dag": {
-      "outputs": {
-        "artifacts": {
-          "output-metrics-2-metrics": {
-            "artifactSelectors": [
-              {
-                "outputArtifactKey": "output-metrics-2-metrics",
-                "producerSubtask": "for-loop-2"
-              }
-            ]
-          },
-          "output-metrics-metrics": {
-            "artifactSelectors": [
-              {
-                "outputArtifactKey": "metrics",
-                "producerSubtask": "output-metrics"
-              }
-            ]
-          }
-        }
-      },
-      "tasks": {
-        "for-loop-2": {
-          "componentRef": {
-            "name": "comp-for-loop-2"
-          },
-          "parameterIterator": {
-            "itemInput": "pipelinechannel--loop-item-param-1",
-            "items": {
-              "raw": "[1, 2]"
-            }
-          },
-          "taskInfo": {
-            "name": "for-loop-2"
-          }
-        },
-        "output-metrics": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-output-metrics"
-          },
-          "taskInfo": {
-            "name": "output-metrics"
-          }
-        }
-      }
-    },
-    "outputDefinitions": {
-      "artifacts": {
-        "output-metrics-2-metrics": {
-          "artifactType": {
-            "schemaTitle": "system.Metrics",
-            "schemaVersion": "0.0.1"
-          }
-        },
-        "output-metrics-metrics": {
-          "artifactType": {
-            "schemaTitle": "system.Metrics",
-            "schemaVersion": "0.0.1"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.py

@@ -40,4 +40,4 @@ def my_pipeline():
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.yaml

@@ -0,0 +1,147 @@
+pipelineInfo:
+  name: pipeline-with-metrics-outputs
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-output-metrics:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef output_metrics(metrics: Output[Metrics]):\n    \"\"\"Dummy component\
+          \ that outputs metrics with a random accuracy.\"\"\"\n    import random\n\
+          \    result = random.randint(0, 100)\n    metrics.log_metric('accuracy',\
+          \ result)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - output_metrics
+    exec-output-metrics-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef output_metrics(metrics: Output[Metrics]):\n    \"\"\"Dummy component\
+          \ that outputs metrics with a random accuracy.\"\"\"\n    import random\n\
+          \    result = random.randint(0, 100)\n    metrics.log_metric('accuracy',\
+          \ result)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - output_metrics
+components:
+  comp-for-loop-2:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop-item-param-1:
+          parameterType: NUMBER_INTEGER
+    outputDefinitions:
+      artifacts:
+        output-metrics-2-metrics:
+          artifactType:
+            schemaTitle: system.Metrics
+            schemaVersion: 0.0.1
+    dag:
+      tasks:
+        output-metrics-2:
+          taskInfo:
+            name: output-metrics-2
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-output-metrics-2
+      outputs:
+        artifacts:
+          output-metrics-2-metrics:
+            artifactSelectors:
+            - producerSubtask: output-metrics-2
+              outputArtifactKey: metrics
+  comp-output-metrics:
+    outputDefinitions:
+      artifacts:
+        metrics:
+          artifactType:
+            schemaTitle: system.Metrics
+            schemaVersion: 0.0.1
+    executorLabel: exec-output-metrics
+  comp-output-metrics-2:
+    outputDefinitions:
+      artifacts:
+        metrics:
+          artifactType:
+            schemaTitle: system.Metrics
+            schemaVersion: 0.0.1
+    executorLabel: exec-output-metrics-2
+root:
+  outputDefinitions:
+    artifacts:
+      output-metrics-metrics:
+        artifactType:
+          schemaTitle: system.Metrics
+          schemaVersion: 0.0.1
+      output-metrics-2-metrics:
+        artifactType:
+          schemaTitle: system.Metrics
+          schemaVersion: 0.0.1
+  dag:
+    tasks:
+      for-loop-2:
+        taskInfo:
+          name: for-loop-2
+        componentRef:
+          name: comp-for-loop-2
+        parameterIterator:
+          items:
+            raw: '[1, 2]'
+          itemInput: pipelinechannel--loop-item-param-1
+      output-metrics:
+        taskInfo:
+          name: output-metrics
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-output-metrics
+    outputs:
+      artifacts:
+        output-metrics-metrics:
+          artifactSelectors:
+          - producerSubtask: output-metrics
+            outputArtifactKey: metrics
+        output-metrics-2-metrics:
+          artifactSelectors:
+          - producerSubtask: for-loop-2
+            outputArtifactKey: output-metrics-2-metrics
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json

@@ -1,495 +0,0 @@
-{
-  "components": {
-    "comp-condition-1": {
-      "dag": {
-        "tasks": {
-          "condition-2": {
-            "componentRef": {
-              "name": "comp-condition-2"
-            },
-            "dependentTasks": [
-              "flip-coin-op-3"
-            ],
-            "inputs": {
-              "parameters": {
-                "pipelinechannel--flip-coin-op-2-Output": {
-                  "componentInputParameter": "pipelinechannel--flip-coin-op-2-Output"
-                },
-                "pipelinechannel--flip-coin-op-3-Output": {
-                  "taskOutputParameter": {
-                    "outputParameterKey": "Output",
-                    "producerTask": "flip-coin-op-3"
-                  }
-                },
-                "pipelinechannel--flip-coin-op-Output": {
-                  "componentInputParameter": "pipelinechannel--flip-coin-op-Output"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "condition-2"
-            },
-            "triggerPolicy": {
-              "condition": "inputs.parameter_values['pipelinechannel--flip-coin-op-2-Output'] == inputs.parameter_values['pipelinechannel--flip-coin-op-3-Output']"
-            }
-          },
-          "flip-coin-op-3": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-flip-coin-op-3"
-            },
-            "taskInfo": {
-              "name": "flip-coin-op-3"
-            }
-          },
-          "print-op-3": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op-3"
-            },
-            "dependentTasks": [
-              "flip-coin-op-3"
-            ],
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "taskOutputParameter": {
-                    "outputParameterKey": "Output",
-                    "producerTask": "flip-coin-op-3"
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op-3"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--flip-coin-op-2-Output": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--flip-coin-op-Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-condition-2": {
-      "dag": {
-        "tasks": {
-          "flip-coin-op-4": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-flip-coin-op-4"
-            },
-            "taskInfo": {
-              "name": "flip-coin-op-4"
-            }
-          },
-          "print-op-4": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op-4"
-            },
-            "dependentTasks": [
-              "flip-coin-op-4"
-            ],
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "taskOutputParameter": {
-                    "outputParameterKey": "Output",
-                    "producerTask": "flip-coin-op-4"
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op-4"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--flip-coin-op-2-Output": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--flip-coin-op-3-Output": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--flip-coin-op-Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-flip-coin-op": {
-      "executorLabel": "exec-flip-coin-op",
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-flip-coin-op-2": {
-      "executorLabel": "exec-flip-coin-op-2",
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-flip-coin-op-3": {
-      "executorLabel": "exec-flip-coin-op-3",
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-flip-coin-op-4": {
-      "executorLabel": "exec-flip-coin-op-4",
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op": {
-      "executorLabel": "exec-print-op",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-2": {
-      "executorLabel": "exec-print-op-2",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-3": {
-      "executorLabel": "exec-print-op-3",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-4": {
-      "executorLabel": "exec-print-op-4",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-flip-coin-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "flip_coin_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0, 1) == 0 else 'tails'\n    return result\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-flip-coin-op-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "flip_coin_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0, 1) == 0 else 'tails'\n    return result\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-flip-coin-op-3": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "flip_coin_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0, 1) == 0 else 'tails'\n    return result\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-flip-coin-op-4": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "flip_coin_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0, 1) == 0 else 'tails'\n    return result\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-3": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-4": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "nested-conditions-pipeline"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "condition-1": {
-          "componentRef": {
-            "name": "comp-condition-1"
-          },
-          "dependentTasks": [
-            "flip-coin-op",
-            "flip-coin-op-2"
-          ],
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--flip-coin-op-2-Output": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "Output",
-                  "producerTask": "flip-coin-op-2"
-                }
-              },
-              "pipelinechannel--flip-coin-op-Output": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "Output",
-                  "producerTask": "flip-coin-op"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "condition-1"
-          },
-          "triggerPolicy": {
-            "condition": "inputs.parameter_values['pipelinechannel--flip-coin-op-Output'] != 'no-such-result'"
-          }
-        },
-        "flip-coin-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-flip-coin-op"
-          },
-          "taskInfo": {
-            "name": "flip-coin-op"
-          }
-        },
-        "flip-coin-op-2": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-flip-coin-op-2"
-          },
-          "taskInfo": {
-            "name": "flip-coin-op-2"
-          }
-        },
-        "print-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op"
-          },
-          "dependentTasks": [
-            "flip-coin-op"
-          ],
-          "inputs": {
-            "parameters": {
-              "msg": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "Output",
-                  "producerTask": "flip-coin-op"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op"
-          }
-        },
-        "print-op-2": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op-2"
-          },
-          "dependentTasks": [
-            "flip-coin-op-2"
-          ],
-          "inputs": {
-            "parameters": {
-              "msg": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "Output",
-                  "producerTask": "flip-coin-op-2"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op-2"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_nested_conditions.py

@@ -51,4 +51,4 @@ def my_pipeline():
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_nested_conditions.yaml

@@ -0,0 +1,427 @@
+pipelineInfo:
+  name: nested-conditions-pipeline
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-flip-coin-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads\
+          \ or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0,\
+          \ 1) == 0 else 'tails'\n    return result\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - flip_coin_op
+    exec-print-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-flip-coin-op-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads\
+          \ or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0,\
+          \ 1) == 0 else 'tails'\n    return result\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - flip_coin_op
+    exec-print-op-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-flip-coin-op-3:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads\
+          \ or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0,\
+          \ 1) == 0 else 'tails'\n    return result\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - flip_coin_op
+    exec-print-op-3:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-flip-coin-op-4:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef flip_coin_op() -> str:\n    \"\"\"Flip a coin and output heads\
+          \ or tails randomly.\"\"\"\n    import random\n    result = 'heads' if random.randint(0,\
+          \ 1) == 0 else 'tails'\n    return result\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - flip_coin_op
+    exec-print-op-4:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str):\n    \"\"\"Print a message.\"\"\"\n    print(msg)\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+components:
+  comp-condition-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-2-Output:
+          parameterType: STRING
+    dag:
+      tasks:
+        condition-2:
+          taskInfo:
+            name: condition-2
+          inputs:
+            parameters:
+              pipelinechannel--flip-coin-op-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-Output
+              pipelinechannel--flip-coin-op-3-Output:
+                taskOutputParameter:
+                  producerTask: flip-coin-op-3
+                  outputParameterKey: Output
+              pipelinechannel--flip-coin-op-2-Output:
+                componentInputParameter: pipelinechannel--flip-coin-op-2-Output
+          dependentTasks:
+          - flip-coin-op-3
+          componentRef:
+            name: comp-condition-2
+          triggerPolicy:
+            condition: inputs.parameter_values['pipelinechannel--flip-coin-op-2-Output']
+              == inputs.parameter_values['pipelinechannel--flip-coin-op-3-Output']
+        flip-coin-op-3:
+          taskInfo:
+            name: flip-coin-op-3
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-flip-coin-op-3
+        print-op-3:
+          taskInfo:
+            name: print-op-3
+          inputs:
+            parameters:
+              msg:
+                taskOutputParameter:
+                  producerTask: flip-coin-op-3
+                  outputParameterKey: Output
+          dependentTasks:
+          - flip-coin-op-3
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op-3
+  comp-flip-coin-op:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-flip-coin-op
+  comp-print-op:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-op
+  comp-flip-coin-op-2:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-flip-coin-op-2
+  comp-print-op-2:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-op-2
+  comp-condition-2:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--flip-coin-op-Output:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-3-Output:
+          parameterType: STRING
+        pipelinechannel--flip-coin-op-2-Output:
+          parameterType: STRING
+    dag:
+      tasks:
+        flip-coin-op-4:
+          taskInfo:
+            name: flip-coin-op-4
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-flip-coin-op-4
+        print-op-4:
+          taskInfo:
+            name: print-op-4
+          inputs:
+            parameters:
+              msg:
+                taskOutputParameter:
+                  producerTask: flip-coin-op-4
+                  outputParameterKey: Output
+          dependentTasks:
+          - flip-coin-op-4
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op-4
+  comp-flip-coin-op-3:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-flip-coin-op-3
+  comp-print-op-3:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-op-3
+  comp-flip-coin-op-4:
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-flip-coin-op-4
+  comp-print-op-4:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-op-4
+root:
+  dag:
+    tasks:
+      condition-1:
+        taskInfo:
+          name: condition-1
+        inputs:
+          parameters:
+            pipelinechannel--flip-coin-op-Output:
+              taskOutputParameter:
+                producerTask: flip-coin-op
+                outputParameterKey: Output
+            pipelinechannel--flip-coin-op-2-Output:
+              taskOutputParameter:
+                producerTask: flip-coin-op-2
+                outputParameterKey: Output
+        dependentTasks:
+        - flip-coin-op
+        - flip-coin-op-2
+        componentRef:
+          name: comp-condition-1
+        triggerPolicy:
+          condition: inputs.parameter_values['pipelinechannel--flip-coin-op-Output']
+            != 'no-such-result'
+      flip-coin-op:
+        taskInfo:
+          name: flip-coin-op
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-flip-coin-op
+      print-op:
+        taskInfo:
+          name: print-op
+        inputs:
+          parameters:
+            msg:
+              taskOutputParameter:
+                producerTask: flip-coin-op
+                outputParameterKey: Output
+        dependentTasks:
+        - flip-coin-op
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op
+      flip-coin-op-2:
+        taskInfo:
+          name: flip-coin-op-2
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-flip-coin-op-2
+      print-op-2:
+        taskInfo:
+          name: print-op-2
+        inputs:
+          parameters:
+            msg:
+              taskOutputParameter:
+                producerTask: flip-coin-op-2
+                outputParameterKey: Output
+        dependentTasks:
+        - flip-coin-op-2
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op-2

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json

@@ -1,540 +0,0 @@
-{
-  "components": {
-    "comp-condition-1": {
-      "dag": {
-        "tasks": {
-          "condition-2": {
-            "componentRef": {
-              "name": "comp-condition-2"
-            },
-            "dependentTasks": [
-              "generate-random-number"
-            ],
-            "inputs": {
-              "parameters": {
-                "pipelinechannel--flip-coin-output": {
-                  "componentInputParameter": "pipelinechannel--flip-coin-output"
-                },
-                "pipelinechannel--generate-random-number-output": {
-                  "taskOutputParameter": {
-                    "outputParameterKey": "output",
-                    "producerTask": "generate-random-number"
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "condition-2"
-            },
-            "triggerPolicy": {
-              "condition": "int(inputs.parameter_values['pipelinechannel--generate-random-number-output']) > 5"
-            }
-          },
-          "condition-3": {
-            "componentRef": {
-              "name": "comp-condition-3"
-            },
-            "dependentTasks": [
-              "generate-random-number"
-            ],
-            "inputs": {
-              "parameters": {
-                "pipelinechannel--flip-coin-output": {
-                  "componentInputParameter": "pipelinechannel--flip-coin-output"
-                },
-                "pipelinechannel--generate-random-number-output": {
-                  "taskOutputParameter": {
-                    "outputParameterKey": "output",
-                    "producerTask": "generate-random-number"
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "condition-3"
-            },
-            "triggerPolicy": {
-              "condition": "int(inputs.parameter_values['pipelinechannel--generate-random-number-output']) <= 5"
-            }
-          },
-          "generate-random-number": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-generate-random-number"
-            },
-            "taskInfo": {
-              "name": "generate-random-number"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--flip-coin-output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-condition-2": {
-      "dag": {
-        "tasks": {
-          "print": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "runtimeValue": {
-                    "constant": "heads and {{$.inputs.parameters['pipelinechannel--generate-random-number-output']}} > 5!"
-                  }
-                },
-                "pipelinechannel--generate-random-number-output": {
-                  "componentInputParameter": "pipelinechannel--generate-random-number-output"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--flip-coin-output": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--generate-random-number-output": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-condition-3": {
-      "dag": {
-        "tasks": {
-          "print-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-2"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "runtimeValue": {
-                    "constant": "heads and {{$.inputs.parameters['pipelinechannel--generate-random-number-output']}} <= 5!"
-                  }
-                },
-                "pipelinechannel--generate-random-number-output": {
-                  "componentInputParameter": "pipelinechannel--generate-random-number-output"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-2"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--flip-coin-output": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--generate-random-number-output": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-condition-4": {
-      "dag": {
-        "tasks": {
-          "condition-5": {
-            "componentRef": {
-              "name": "comp-condition-5"
-            },
-            "dependentTasks": [
-              "generate-random-number-2"
-            ],
-            "inputs": {
-              "parameters": {
-                "pipelinechannel--flip-coin-output": {
-                  "componentInputParameter": "pipelinechannel--flip-coin-output"
-                },
-                "pipelinechannel--generate-random-number-2-output": {
-                  "taskOutputParameter": {
-                    "outputParameterKey": "output",
-                    "producerTask": "generate-random-number-2"
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "condition-5"
-            },
-            "triggerPolicy": {
-              "condition": "int(inputs.parameter_values['pipelinechannel--generate-random-number-2-output']) > 15"
-            }
-          },
-          "condition-6": {
-            "componentRef": {
-              "name": "comp-condition-6"
-            },
-            "dependentTasks": [
-              "generate-random-number-2"
-            ],
-            "inputs": {
-              "parameters": {
-                "pipelinechannel--flip-coin-output": {
-                  "componentInputParameter": "pipelinechannel--flip-coin-output"
-                },
-                "pipelinechannel--generate-random-number-2-output": {
-                  "taskOutputParameter": {
-                    "outputParameterKey": "output",
-                    "producerTask": "generate-random-number-2"
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "condition-6"
-            },
-            "triggerPolicy": {
-              "condition": "int(inputs.parameter_values['pipelinechannel--generate-random-number-2-output']) <= 15"
-            }
-          },
-          "generate-random-number-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-generate-random-number-2"
-            },
-            "taskInfo": {
-              "name": "generate-random-number-2"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--flip-coin-output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-condition-5": {
-      "dag": {
-        "tasks": {
-          "print-3": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-3"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "runtimeValue": {
-                    "constant": "tails and {{$.inputs.parameters['pipelinechannel--generate-random-number-2-output']}} > 15!"
-                  }
-                },
-                "pipelinechannel--generate-random-number-2-output": {
-                  "componentInputParameter": "pipelinechannel--generate-random-number-2-output"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-3"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--flip-coin-output": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--generate-random-number-2-output": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-condition-6": {
-      "dag": {
-        "tasks": {
-          "print-4": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-4"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "runtimeValue": {
-                    "constant": "tails and {{$.inputs.parameters['pipelinechannel--generate-random-number-2-output']}} <= 15!"
-                  }
-                },
-                "pipelinechannel--generate-random-number-2-output": {
-                  "componentInputParameter": "pipelinechannel--generate-random-number-2-output"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-4"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--flip-coin-output": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--generate-random-number-2-output": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-flip-coin": {
-      "executorLabel": "exec-flip-coin",
-      "outputDefinitions": {
-        "parameters": {
-          "output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-generate-random-number": {
-      "executorLabel": "exec-generate-random-number",
-      "outputDefinitions": {
-        "parameters": {
-          "output": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-generate-random-number-2": {
-      "executorLabel": "exec-generate-random-number-2",
-      "outputDefinitions": {
-        "parameters": {
-          "output": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-print": {
-      "executorLabel": "exec-print",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-2": {
-      "executorLabel": "exec-print-2",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-3": {
-      "executorLabel": "exec-print-3",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-4": {
-      "executorLabel": "exec-print-4",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-flip-coin": {
-        "container": {
-          "args": [
-            "mkdir -p \"$(dirname $0)\" && python -c \"import random; result = 'heads' if random.randint(0,1) == 0 else 'tails'; print(result, end='')\" | tee $0",
-            "{{$.outputs.parameters['output'].output_file}}"
-          ],
-          "command": [
-            "sh",
-            "-c"
-          ],
-          "image": "python:alpine3.6"
-        }
-      },
-      "exec-generate-random-number": {
-        "container": {
-          "args": [
-            "mkdir -p \"$(dirname $2)\" && python -c \"import random; print(random.randint($0, $1), end='')\" | tee $2",
-            "0",
-            "9",
-            "{{$.outputs.parameters['output'].output_file}}"
-          ],
-          "command": [
-            "sh",
-            "-c"
-          ],
-          "image": "python:alpine3.6"
-        }
-      },
-      "exec-generate-random-number-2": {
-        "container": {
-          "args": [
-            "mkdir -p \"$(dirname $2)\" && python -c \"import random; print(random.randint($0, $1), end='')\" | tee $2",
-            "10",
-            "19",
-            "{{$.outputs.parameters['output'].output_file}}"
-          ],
-          "command": [
-            "sh",
-            "-c"
-          ],
-          "image": "python:alpine3.6"
-        }
-      },
-      "exec-print": {
-        "container": {
-          "command": [
-            "echo",
-            "{{$.inputs.parameters['msg']}}"
-          ],
-          "image": "python:alpine3.6"
-        }
-      },
-      "exec-print-2": {
-        "container": {
-          "command": [
-            "echo",
-            "{{$.inputs.parameters['msg']}}"
-          ],
-          "image": "python:alpine3.6"
-        }
-      },
-      "exec-print-3": {
-        "container": {
-          "command": [
-            "echo",
-            "{{$.inputs.parameters['msg']}}"
-          ],
-          "image": "python:alpine3.6"
-        }
-      },
-      "exec-print-4": {
-        "container": {
-          "command": [
-            "echo",
-            "{{$.inputs.parameters['msg']}}"
-          ],
-          "image": "python:alpine3.6"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "conditional-execution-pipeline"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "condition-1": {
-          "componentRef": {
-            "name": "comp-condition-1"
-          },
-          "dependentTasks": [
-            "flip-coin"
-          ],
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--flip-coin-output": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "output",
-                  "producerTask": "flip-coin"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "condition-1"
-          },
-          "triggerPolicy": {
-            "condition": "inputs.parameter_values['pipelinechannel--flip-coin-output'] == 'heads'"
-          }
-        },
-        "condition-4": {
-          "componentRef": {
-            "name": "comp-condition-4"
-          },
-          "dependentTasks": [
-            "flip-coin"
-          ],
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--flip-coin-output": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "output",
-                  "producerTask": "flip-coin"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "condition-4"
-          },
-          "triggerPolicy": {
-            "condition": "inputs.parameter_values['pipelinechannel--flip-coin-output'] == 'tails'"
-          }
-        },
-        "flip-coin": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-flip-coin"
-          },
-          "taskInfo": {
-            "name": "flip-coin"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.py

@@ -89,4 +89,4 @@ def my_pipeline():
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.yaml

@@ -0,0 +1,346 @@
+pipelineInfo:
+  name: conditional-execution-pipeline
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-flip-coin:
+      container:
+        image: python:alpine3.6
+        command:
+        - sh
+        - -c
+        args:
+        - mkdir -p "$(dirname $0)" && python -c "import random; result = 'heads' if
+          random.randint(0,1) == 0 else 'tails'; print(result, end='')" | tee $0
+        - '{{$.outputs.parameters[''output''].output_file}}'
+    exec-generate-random-number:
+      container:
+        image: python:alpine3.6
+        command:
+        - sh
+        - -c
+        args:
+        - mkdir -p "$(dirname $2)" && python -c "import random; print(random.randint($0,
+          $1), end='')" | tee $2
+        - '0'
+        - '9'
+        - '{{$.outputs.parameters[''output''].output_file}}'
+    exec-print:
+      container:
+        image: python:alpine3.6
+        command:
+        - echo
+        - '{{$.inputs.parameters[''msg'']}}'
+    exec-print-2:
+      container:
+        image: python:alpine3.6
+        command:
+        - echo
+        - '{{$.inputs.parameters[''msg'']}}'
+    exec-generate-random-number-2:
+      container:
+        image: python:alpine3.6
+        command:
+        - sh
+        - -c
+        args:
+        - mkdir -p "$(dirname $2)" && python -c "import random; print(random.randint($0,
+          $1), end='')" | tee $2
+        - '10'
+        - '19'
+        - '{{$.outputs.parameters[''output''].output_file}}'
+    exec-print-3:
+      container:
+        image: python:alpine3.6
+        command:
+        - echo
+        - '{{$.inputs.parameters[''msg'']}}'
+    exec-print-4:
+      container:
+        image: python:alpine3.6
+        command:
+        - echo
+        - '{{$.inputs.parameters[''msg'']}}'
+components:
+  comp-condition-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--flip-coin-output:
+          parameterType: STRING
+    dag:
+      tasks:
+        condition-2:
+          taskInfo:
+            name: condition-2
+          inputs:
+            parameters:
+              pipelinechannel--flip-coin-output:
+                componentInputParameter: pipelinechannel--flip-coin-output
+              pipelinechannel--generate-random-number-output:
+                taskOutputParameter:
+                  producerTask: generate-random-number
+                  outputParameterKey: output
+          dependentTasks:
+          - generate-random-number
+          componentRef:
+            name: comp-condition-2
+          triggerPolicy:
+            condition: int(inputs.parameter_values['pipelinechannel--generate-random-number-output'])
+              > 5
+        condition-3:
+          taskInfo:
+            name: condition-3
+          inputs:
+            parameters:
+              pipelinechannel--flip-coin-output:
+                componentInputParameter: pipelinechannel--flip-coin-output
+              pipelinechannel--generate-random-number-output:
+                taskOutputParameter:
+                  producerTask: generate-random-number
+                  outputParameterKey: output
+          dependentTasks:
+          - generate-random-number
+          componentRef:
+            name: comp-condition-3
+          triggerPolicy:
+            condition: int(inputs.parameter_values['pipelinechannel--generate-random-number-output'])
+              <= 5
+        generate-random-number:
+          taskInfo:
+            name: generate-random-number
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-generate-random-number
+  comp-condition-4:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--flip-coin-output:
+          parameterType: STRING
+    dag:
+      tasks:
+        condition-5:
+          taskInfo:
+            name: condition-5
+          inputs:
+            parameters:
+              pipelinechannel--flip-coin-output:
+                componentInputParameter: pipelinechannel--flip-coin-output
+              pipelinechannel--generate-random-number-2-output:
+                taskOutputParameter:
+                  producerTask: generate-random-number-2
+                  outputParameterKey: output
+          dependentTasks:
+          - generate-random-number-2
+          componentRef:
+            name: comp-condition-5
+          triggerPolicy:
+            condition: int(inputs.parameter_values['pipelinechannel--generate-random-number-2-output'])
+              > 15
+        condition-6:
+          taskInfo:
+            name: condition-6
+          inputs:
+            parameters:
+              pipelinechannel--flip-coin-output:
+                componentInputParameter: pipelinechannel--flip-coin-output
+              pipelinechannel--generate-random-number-2-output:
+                taskOutputParameter:
+                  producerTask: generate-random-number-2
+                  outputParameterKey: output
+          dependentTasks:
+          - generate-random-number-2
+          componentRef:
+            name: comp-condition-6
+          triggerPolicy:
+            condition: int(inputs.parameter_values['pipelinechannel--generate-random-number-2-output'])
+              <= 15
+        generate-random-number-2:
+          taskInfo:
+            name: generate-random-number-2
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-generate-random-number-2
+  comp-flip-coin:
+    outputDefinitions:
+      parameters:
+        output:
+          parameterType: STRING
+    executorLabel: exec-flip-coin
+  comp-condition-2:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--flip-coin-output:
+          parameterType: STRING
+        pipelinechannel--generate-random-number-output:
+          parameterType: NUMBER_INTEGER
+    dag:
+      tasks:
+        print:
+          taskInfo:
+            name: print
+          inputs:
+            parameters:
+              pipelinechannel--generate-random-number-output:
+                componentInputParameter: pipelinechannel--generate-random-number-output
+              msg:
+                runtimeValue:
+                  constant: heads and {{$.inputs.parameters['pipelinechannel--generate-random-number-output']}}
+                    > 5!
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print
+  comp-condition-3:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--flip-coin-output:
+          parameterType: STRING
+        pipelinechannel--generate-random-number-output:
+          parameterType: NUMBER_INTEGER
+    dag:
+      tasks:
+        print-2:
+          taskInfo:
+            name: print-2
+          inputs:
+            parameters:
+              pipelinechannel--generate-random-number-output:
+                componentInputParameter: pipelinechannel--generate-random-number-output
+              msg:
+                runtimeValue:
+                  constant: heads and {{$.inputs.parameters['pipelinechannel--generate-random-number-output']}}
+                    <= 5!
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-2
+  comp-generate-random-number:
+    outputDefinitions:
+      parameters:
+        output:
+          parameterType: NUMBER_INTEGER
+    executorLabel: exec-generate-random-number
+  comp-print:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print
+  comp-print-2:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-2
+  comp-condition-5:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--flip-coin-output:
+          parameterType: STRING
+        pipelinechannel--generate-random-number-2-output:
+          parameterType: NUMBER_INTEGER
+    dag:
+      tasks:
+        print-3:
+          taskInfo:
+            name: print-3
+          inputs:
+            parameters:
+              pipelinechannel--generate-random-number-2-output:
+                componentInputParameter: pipelinechannel--generate-random-number-2-output
+              msg:
+                runtimeValue:
+                  constant: tails and {{$.inputs.parameters['pipelinechannel--generate-random-number-2-output']}}
+                    > 15!
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-3
+  comp-condition-6:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--flip-coin-output:
+          parameterType: STRING
+        pipelinechannel--generate-random-number-2-output:
+          parameterType: NUMBER_INTEGER
+    dag:
+      tasks:
+        print-4:
+          taskInfo:
+            name: print-4
+          inputs:
+            parameters:
+              pipelinechannel--generate-random-number-2-output:
+                componentInputParameter: pipelinechannel--generate-random-number-2-output
+              msg:
+                runtimeValue:
+                  constant: tails and {{$.inputs.parameters['pipelinechannel--generate-random-number-2-output']}}
+                    <= 15!
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-4
+  comp-generate-random-number-2:
+    outputDefinitions:
+      parameters:
+        output:
+          parameterType: NUMBER_INTEGER
+    executorLabel: exec-generate-random-number-2
+  comp-print-3:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-3
+  comp-print-4:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-4
+root:
+  dag:
+    tasks:
+      condition-1:
+        taskInfo:
+          name: condition-1
+        inputs:
+          parameters:
+            pipelinechannel--flip-coin-output:
+              taskOutputParameter:
+                producerTask: flip-coin
+                outputParameterKey: output
+        dependentTasks:
+        - flip-coin
+        componentRef:
+          name: comp-condition-1
+        triggerPolicy:
+          condition: inputs.parameter_values['pipelinechannel--flip-coin-output']
+            == 'heads'
+      condition-4:
+        taskInfo:
+          name: condition-4
+        inputs:
+          parameters:
+            pipelinechannel--flip-coin-output:
+              taskOutputParameter:
+                producerTask: flip-coin
+                outputParameterKey: output
+        dependentTasks:
+        - flip-coin
+        componentRef:
+          name: comp-condition-4
+        triggerPolicy:
+          condition: inputs.parameter_values['pipelinechannel--flip-coin-output']
+            == 'tails'
+      flip-coin:
+        taskInfo:
+          name: flip-coin
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-flip-coin
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_nested_loops.json

@@ -1,346 +0,0 @@
-{
-  "components": {
-    "comp-for-loop-1": {
-      "dag": {
-        "tasks": {
-          "for-loop-2": {
-            "componentRef": {
-              "name": "comp-for-loop-2"
-            },
-            "inputs": {
-              "parameters": {
-                "pipelinechannel--loop_parameter-loop-item": {
-                  "componentInputParameter": "pipelinechannel--loop_parameter-loop-item"
-                },
-                "pipelinechannel--loop_parameter-loop-item-subvar-p_a": {
-                  "componentInputParameter": "pipelinechannel--loop_parameter-loop-item",
-                  "parameterExpressionSelector": "parseJson(string_value)[\"p_a\"]"
-                }
-              }
-            },
-            "parameterIterator": {
-              "itemInput": "pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item",
-              "items": {
-                "inputParameter": "pipelinechannel--loop_parameter-loop-item-subvar-p_a"
-              }
-            },
-            "taskInfo": {
-              "name": "for-loop-2"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--loop_parameter": {
-            "parameterType": "LIST"
-          },
-          "pipelinechannel--loop_parameter-loop-item": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-for-loop-2": {
-      "dag": {
-        "tasks": {
-          "print-op": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "componentInputParameter": "pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item",
-                  "parameterExpressionSelector": "parseJson(string_value)[\"q_a\"]"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--loop_parameter-loop-item": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--loop_parameter-loop-item-subvar-p_a": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-for-loop-4": {
-      "dag": {
-        "tasks": {
-          "for-loop-6": {
-            "componentRef": {
-              "name": "comp-for-loop-6"
-            },
-            "inputs": {
-              "parameters": {
-                "pipelinechannel--loop-item-param-3": {
-                  "componentInputParameter": "pipelinechannel--loop-item-param-3"
-                }
-              }
-            },
-            "parameterIterator": {
-              "itemInput": "pipelinechannel--loop-item-param-5",
-              "items": {
-                "raw": "[\"100\", \"200\", \"300\"]"
-              }
-            },
-            "taskInfo": {
-              "name": "for-loop-6"
-            }
-          },
-          "print-op-2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op-2"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "componentInputParameter": "pipelinechannel--loop-item-param-3"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op-2"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--loop-item-param-3": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-for-loop-6": {
-      "dag": {
-        "tasks": {
-          "print-op-3": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op-3"
-            },
-            "inputs": {
-              "parameters": {
-                "msg": {
-                  "componentInputParameter": "pipelinechannel--loop-item-param-3"
-                },
-                "msg2": {
-                  "componentInputParameter": "pipelinechannel--loop-item-param-5"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op-3"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--loop-item-param-3": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--loop-item-param-5": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op": {
-      "executorLabel": "exec-print-op",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-2": {
-      "executorLabel": "exec-print-op-2",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-3": {
-      "executorLabel": "exec-print-op-3",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          },
-          "msg2": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-print-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n    print(f'msg: {msg}, msg2: {msg2}')\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n    print(f'msg: {msg}, msg2: {msg2}')\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-3": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n    print(f'msg: {msg}, msg2: {msg2}')\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-nested-loops"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "for-loop-1": {
-          "componentRef": {
-            "name": "comp-for-loop-1"
-          },
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--loop_parameter": {
-                "componentInputParameter": "loop_parameter"
-              }
-            }
-          },
-          "parameterIterator": {
-            "itemInput": "pipelinechannel--loop_parameter-loop-item",
-            "items": {
-              "inputParameter": "pipelinechannel--loop_parameter"
-            }
-          },
-          "taskInfo": {
-            "name": "for-loop-1"
-          }
-        },
-        "for-loop-4": {
-          "componentRef": {
-            "name": "comp-for-loop-4"
-          },
-          "parameterIterator": {
-            "itemInput": "pipelinechannel--loop-item-param-3",
-            "items": {
-              "raw": "[\"1\", \"2\"]"
-            }
-          },
-          "taskInfo": {
-            "name": "for-loop-4"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "loop_parameter": {
-          "defaultValue": [
-            {
-              "p_a": [
-                {
-                  "q_a": "1"
-                },
-                {
-                  "q_a": "2"
-                }
-              ],
-              "p_b": "hello"
-            },
-            {
-              "p_a": [
-                {
-                  "q_a": "11"
-                },
-                {
-                  "q_a": "22"
-                }
-              ],
-              "p_b": "halo"
-            }
-          ],
-          "parameterType": "LIST"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_nested_loops.py

@@ -57,4 +57,4 @@ def my_pipeline(loop_parameter: list = [
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_nested_loops.yaml

@@ -0,0 +1,247 @@
+pipelineInfo:
+  name: pipeline-with-nested-loops
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-print-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-print-op-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-print-op-3:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n    print(f'msg:\
+          \ {msg}, msg2: {msg2}')\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+components:
+  comp-for-loop-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop_parameter:
+          parameterType: LIST
+        pipelinechannel--loop_parameter-loop-item:
+          parameterType: STRING
+    dag:
+      tasks:
+        for-loop-2:
+          taskInfo:
+            name: for-loop-2
+          inputs:
+            parameters:
+              pipelinechannel--loop_parameter-loop-item:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+              pipelinechannel--loop_parameter-loop-item-subvar-p_a:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item
+                parameterExpressionSelector: parseJson(string_value)["p_a"]
+          componentRef:
+            name: comp-for-loop-2
+          parameterIterator:
+            items:
+              inputParameter: pipelinechannel--loop_parameter-loop-item-subvar-p_a
+            itemInput: pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item
+  comp-for-loop-4:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop-item-param-3:
+          parameterType: STRING
+    dag:
+      tasks:
+        for-loop-6:
+          taskInfo:
+            name: for-loop-6
+          inputs:
+            parameters:
+              pipelinechannel--loop-item-param-3:
+                componentInputParameter: pipelinechannel--loop-item-param-3
+          componentRef:
+            name: comp-for-loop-6
+          parameterIterator:
+            items:
+              raw: '["100", "200", "300"]'
+            itemInput: pipelinechannel--loop-item-param-5
+        print-op-2:
+          taskInfo:
+            name: print-op-2
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop-item-param-3
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op-2
+  comp-for-loop-2:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop_parameter-loop-item:
+          parameterType: STRING
+        pipelinechannel--loop_parameter-loop-item-subvar-p_a:
+          parameterType: STRING
+        pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-op:
+          taskInfo:
+            name: print-op
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item
+                parameterExpressionSelector: parseJson(string_value)["q_a"]
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op
+  comp-print-op:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-op
+  comp-for-loop-6:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--loop-item-param-5:
+          parameterType: STRING
+        pipelinechannel--loop-item-param-3:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-op-3:
+          taskInfo:
+            name: print-op-3
+          inputs:
+            parameters:
+              msg:
+                componentInputParameter: pipelinechannel--loop-item-param-3
+              msg2:
+                componentInputParameter: pipelinechannel--loop-item-param-5
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op-3
+  comp-print-op-2:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+    executorLabel: exec-print-op-2
+  comp-print-op-3:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+        msg2:
+          parameterType: STRING
+    executorLabel: exec-print-op-3
+root:
+  inputDefinitions:
+    parameters:
+      loop_parameter:
+        parameterType: LIST
+        defaultValue:
+        - p_a:
+          - q_a: '1'
+          - q_a: '2'
+          p_b: hello
+        - p_a:
+          - q_a: '11'
+          - q_a: '22'
+          p_b: halo
+  dag:
+    tasks:
+      for-loop-1:
+        taskInfo:
+          name: for-loop-1
+        inputs:
+          parameters:
+            pipelinechannel--loop_parameter:
+              componentInputParameter: loop_parameter
+        componentRef:
+          name: comp-for-loop-1
+        parameterIterator:
+          items:
+            inputParameter: pipelinechannel--loop_parameter
+          itemInput: pipelinechannel--loop_parameter-loop-item
+      for-loop-4:
+        taskInfo:
+          name: for-loop-4
+        componentRef:
+          name: comp-for-loop-4
+        parameterIterator:
+          items:
+            raw: '["1", "2"]'
+          itemInput: pipelinechannel--loop-item-param-3

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_ontology.json

@@ -1,163 +0,0 @@
-{
-  "components": {
-    "comp-ingestion": {
-      "executorLabel": "exec-ingestion",
-      "inputDefinitions": {
-        "parameters": {
-          "input_location": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "examples": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    },
-    "comp-trainer": {
-      "executorLabel": "exec-trainer",
-      "inputDefinitions": {
-        "artifacts": {
-          "examples": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        },
-        "parameters": {
-          "n_epochs": {
-            "parameterType": "NUMBER_INTEGER"
-          },
-          "optimizer": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "model": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-ingestion": {
-        "container": {
-          "args": [
-            "--input-location",
-            "{{$.inputs.parameters['input_location']}}",
-            "--output-examples",
-            "{{$.outputs.artifacts['examples'].uri}}"
-          ],
-          "image": "gcr.io/my-project/my-ingestor"
-        }
-      },
-      "exec-trainer": {
-        "container": {
-          "args": [
-            "--input-examples",
-            "{{$.inputs.artifacts['examples'].uri}}",
-            "--optimizer",
-            "{{$.inputs.parameters['optimizer']}}",
-            "--n_epochs",
-            "{{$.inputs.parameters['n_epochs']}}",
-            "--output-model",
-            "{{$.outputs.artifacts['model'].uri}}"
-          ],
-          "image": "gcr.io/my-project/my-fancy-trainer"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "two-step-pipeline-with-ontology"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "ingestion": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-ingestion"
-          },
-          "inputs": {
-            "parameters": {
-              "input_location": {
-                "componentInputParameter": "input_location"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "ingestion"
-          }
-        },
-        "trainer": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-trainer"
-          },
-          "dependentTasks": [
-            "ingestion"
-          ],
-          "inputs": {
-            "artifacts": {
-              "examples": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "examples",
-                  "producerTask": "ingestion"
-                }
-              }
-            },
-            "parameters": {
-              "n_epochs": {
-                "componentInputParameter": "n_epochs"
-              },
-              "optimizer": {
-                "componentInputParameter": "optimizer"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "trainer"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "input_location": {
-          "defaultValue": "gs://test-bucket/pipeline_root",
-          "parameterType": "STRING"
-        },
-        "n_epochs": {
-          "defaultValue": 200.0,
-          "parameterType": "NUMBER_INTEGER"
-        },
-        "optimizer": {
-          "defaultValue": "sgd",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_ontology.py

@@ -43,4 +43,4 @@ def my_pipeline(input_location: str = 'gs://test-bucket/pipeline_root',
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_ontology.yaml

@@ -0,0 +1,104 @@
+pipelineInfo:
+  name: two-step-pipeline-with-ontology
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-ingestion:
+      container:
+        image: gcr.io/my-project/my-ingestor
+        args:
+        - --input-location
+        - '{{$.inputs.parameters[''input_location'']}}'
+        - --output-examples
+        - '{{$.outputs.artifacts[''examples''].uri}}'
+    exec-trainer:
+      container:
+        image: gcr.io/my-project/my-fancy-trainer
+        args:
+        - --input-examples
+        - '{{$.inputs.artifacts[''examples''].uri}}'
+        - --optimizer
+        - '{{$.inputs.parameters[''optimizer'']}}'
+        - --n_epochs
+        - '{{$.inputs.parameters[''n_epochs'']}}'
+        - --output-model
+        - '{{$.outputs.artifacts[''model''].uri}}'
+components:
+  comp-ingestion:
+    inputDefinitions:
+      parameters:
+        input_location:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        examples:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+    executorLabel: exec-ingestion
+  comp-trainer:
+    inputDefinitions:
+      artifacts:
+        examples:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+      parameters:
+        optimizer:
+          parameterType: STRING
+        n_epochs:
+          parameterType: NUMBER_INTEGER
+    outputDefinitions:
+      artifacts:
+        model:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+    executorLabel: exec-trainer
+root:
+  inputDefinitions:
+    parameters:
+      input_location:
+        parameterType: STRING
+        defaultValue: gs://test-bucket/pipeline_root
+      optimizer:
+        parameterType: STRING
+        defaultValue: sgd
+      n_epochs:
+        parameterType: NUMBER_INTEGER
+        defaultValue: 200.0
+  dag:
+    tasks:
+      ingestion:
+        taskInfo:
+          name: ingestion
+        inputs:
+          parameters:
+            input_location:
+              componentInputParameter: input_location
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-ingestion
+      trainer:
+        taskInfo:
+          name: trainer
+        inputs:
+          parameters:
+            optimizer:
+              componentInputParameter: optimizer
+            n_epochs:
+              componentInputParameter: n_epochs
+          artifacts:
+            examples:
+              taskOutputArtifact:
+                producerTask: ingestion
+                outputArtifactKey: examples
+        dependentTasks:
+        - ingestion
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-trainer
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json

@@ -1,255 +0,0 @@
-{
-  "components": {
-    "comp-for-loop-2": {
-      "dag": {
-        "tasks": {
-          "print-op2": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op2"
-            },
-            "inputs": {
-              "parameters": {
-                "pipelinechannel--name": {
-                  "componentInputParameter": "pipelinechannel--name"
-                },
-                "text1": {
-                  "componentInputParameter": "pipelinechannel--loop-item-param-1"
-                },
-                "text2": {
-                  "runtimeValue": {
-                    "constant": " and {{$.inputs.parameters['pipelinechannel--name']}}."
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op2"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--loop-item-param-1": {
-            "parameterType": "STRING"
-          },
-          "pipelinechannel--name": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op": {
-      "executorLabel": "exec-print-op",
-      "inputDefinitions": {
-        "parameters": {
-          "text": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-2": {
-      "executorLabel": "exec-print-op-2",
-      "inputDefinitions": {
-        "parameters": {
-          "text": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op2": {
-      "executorLabel": "exec-print-op2",
-      "inputDefinitions": {
-        "parameters": {
-          "text1": {
-            "parameterType": "STRING"
-          },
-          "text2": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "parameters": {
-          "Output": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-print-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(text: str) -> str:\n    print(text)\n    return text\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(text: str) -> str:\n    print(text)\n    return text\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op2"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op2(text1: str, text2: str) -> str:\n    print(text1 + text2)\n    return text1 + text2\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-pipelineparam-containing-format"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "for-loop-2": {
-          "componentRef": {
-            "name": "comp-for-loop-2"
-          },
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--name": {
-                "componentInputParameter": "name"
-              }
-            }
-          },
-          "parameterIterator": {
-            "itemInput": "pipelinechannel--loop-item-param-1",
-            "items": {
-              "raw": "[\"1\", \"2\"]"
-            }
-          },
-          "taskInfo": {
-            "name": "for-loop-2"
-          }
-        },
-        "print-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op"
-          },
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--name": {
-                "componentInputParameter": "name"
-              },
-              "text": {
-                "runtimeValue": {
-                  "constant": "Hello {{$.inputs.parameters['pipelinechannel--name']}}"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op"
-          }
-        },
-        "print-op-2": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op-2"
-          },
-          "dependentTasks": [
-            "print-op"
-          ],
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--print-op-Output": {
-                "taskOutputParameter": {
-                  "producerTask": "print-op"
-                }
-              },
-              "text": {
-                "runtimeValue": {
-                  "constant": "{{$.inputs.parameters['pipelinechannel--print-op-Output']}}, again."
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op-2"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "name": {
-          "defaultValue": "KFP",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_params_containing_format.py

@@ -43,4 +43,4 @@ def my_pipeline(name: str = 'KFP'):
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_params_containing_format.yaml

@@ -0,0 +1,198 @@
+pipelineInfo:
+  name: pipeline-with-pipelineparam-containing-format
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-print-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(text: str) -> str:\n    print(text)\n    return text\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-print-op-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(text: str) -> str:\n    print(text)\n    return text\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-print-op2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op2(text1: str, text2: str) -> str:\n    print(text1 +\
+          \ text2)\n    return text1 + text2\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op2
+components:
+  comp-for-loop-2:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--name:
+          parameterType: STRING
+        pipelinechannel--loop-item-param-1:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-op2:
+          taskInfo:
+            name: print-op2
+          inputs:
+            parameters:
+              text1:
+                componentInputParameter: pipelinechannel--loop-item-param-1
+              pipelinechannel--name:
+                componentInputParameter: pipelinechannel--name
+              text2:
+                runtimeValue:
+                  constant: ' and {{$.inputs.parameters[''pipelinechannel--name'']}}.'
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op2
+  comp-print-op:
+    inputDefinitions:
+      parameters:
+        text:
+          parameterType: STRING
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-print-op
+  comp-print-op-2:
+    inputDefinitions:
+      parameters:
+        text:
+          parameterType: STRING
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-print-op-2
+  comp-print-op2:
+    inputDefinitions:
+      parameters:
+        text1:
+          parameterType: STRING
+        text2:
+          parameterType: STRING
+    outputDefinitions:
+      parameters:
+        Output:
+          parameterType: STRING
+    executorLabel: exec-print-op2
+root:
+  inputDefinitions:
+    parameters:
+      name:
+        parameterType: STRING
+        defaultValue: KFP
+  dag:
+    tasks:
+      for-loop-2:
+        taskInfo:
+          name: for-loop-2
+        inputs:
+          parameters:
+            pipelinechannel--name:
+              componentInputParameter: name
+        componentRef:
+          name: comp-for-loop-2
+        parameterIterator:
+          items:
+            raw: '["1", "2"]'
+          itemInput: pipelinechannel--loop-item-param-1
+      print-op:
+        taskInfo:
+          name: print-op
+        inputs:
+          parameters:
+            pipelinechannel--name:
+              componentInputParameter: name
+            text:
+              runtimeValue:
+                constant: Hello {{$.inputs.parameters['pipelinechannel--name']}}
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op
+      print-op-2:
+        taskInfo:
+          name: print-op-2
+        inputs:
+          parameters:
+            pipelinechannel--print-op-Output:
+              taskOutputParameter:
+                producerTask: print-op
+            text:
+              runtimeValue:
+                constant: '{{$.inputs.parameters[''pipelinechannel--print-op-Output'']}},
+                  again.'
+        dependentTasks:
+        - print-op
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op-2

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_placeholders.json

@@ -1,309 +0,0 @@
-{
-  "components": {
-    "comp-print-op": {
-      "executorLabel": "exec-print-op",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          },
-          "value": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-2": {
-      "executorLabel": "exec-print-op-2",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          },
-          "value": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-3": {
-      "executorLabel": "exec-print-op-3",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          },
-          "value": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-4": {
-      "executorLabel": "exec-print-op-4",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          },
-          "value": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op-5": {
-      "executorLabel": "exec-print-op-5",
-      "inputDefinitions": {
-        "parameters": {
-          "msg": {
-            "parameterType": "STRING"
-          },
-          "value": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-print-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-2": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-3": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-4": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op-5": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-placeholders"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "print-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op"
-          },
-          "inputs": {
-            "parameters": {
-              "msg": {
-                "runtimeValue": {
-                  "constant": "job name:"
-                }
-              },
-              "value": {
-                "runtimeValue": {
-                  "constant": "{{$.pipeline_job_name}}"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op"
-          }
-        },
-        "print-op-2": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op-2"
-          },
-          "inputs": {
-            "parameters": {
-              "msg": {
-                "runtimeValue": {
-                  "constant": "job resource name:"
-                }
-              },
-              "value": {
-                "runtimeValue": {
-                  "constant": "{{$.pipeline_job_resource_name}}"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op-2"
-          }
-        },
-        "print-op-3": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op-3"
-          },
-          "inputs": {
-            "parameters": {
-              "msg": {
-                "runtimeValue": {
-                  "constant": "job id:"
-                }
-              },
-              "value": {
-                "runtimeValue": {
-                  "constant": "{{$.pipeline_job_uuid}}"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op-3"
-          }
-        },
-        "print-op-4": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op-4"
-          },
-          "inputs": {
-            "parameters": {
-              "msg": {
-                "runtimeValue": {
-                  "constant": "task name:"
-                }
-              },
-              "value": {
-                "runtimeValue": {
-                  "constant": "{{$.pipeline_task_name}}"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op-4"
-          }
-        },
-        "print-op-5": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-print-op-5"
-          },
-          "inputs": {
-            "parameters": {
-              "msg": {
-                "runtimeValue": {
-                  "constant": "task id:"
-                }
-              },
-              "value": {
-                "runtimeValue": {
-                  "constant": "{{$.pipeline_task_uuid}}"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "print-op-5"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_placeholders.py

@@ -49,4 +49,4 @@ def my_pipeline():
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_placeholders.yaml

@@ -0,0 +1,255 @@
+pipelineInfo:
+  name: pipeline-with-placeholders
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-print-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-print-op-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-print-op-3:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-print-op-4:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-print-op-5:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(msg: str, value: str):\n    print(msg, value)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+components:
+  comp-print-op:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+        value:
+          parameterType: STRING
+    executorLabel: exec-print-op
+  comp-print-op-2:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+        value:
+          parameterType: STRING
+    executorLabel: exec-print-op-2
+  comp-print-op-3:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+        value:
+          parameterType: STRING
+    executorLabel: exec-print-op-3
+  comp-print-op-4:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+        value:
+          parameterType: STRING
+    executorLabel: exec-print-op-4
+  comp-print-op-5:
+    inputDefinitions:
+      parameters:
+        msg:
+          parameterType: STRING
+        value:
+          parameterType: STRING
+    executorLabel: exec-print-op-5
+root:
+  dag:
+    tasks:
+      print-op:
+        taskInfo:
+          name: print-op
+        inputs:
+          parameters:
+            msg:
+              runtimeValue:
+                constant: 'job name:'
+            value:
+              runtimeValue:
+                constant: '{{$.pipeline_job_name}}'
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op
+      print-op-2:
+        taskInfo:
+          name: print-op-2
+        inputs:
+          parameters:
+            msg:
+              runtimeValue:
+                constant: 'job resource name:'
+            value:
+              runtimeValue:
+                constant: '{{$.pipeline_job_resource_name}}'
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op-2
+      print-op-3:
+        taskInfo:
+          name: print-op-3
+        inputs:
+          parameters:
+            msg:
+              runtimeValue:
+                constant: 'job id:'
+            value:
+              runtimeValue:
+                constant: '{{$.pipeline_job_uuid}}'
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op-3
+      print-op-4:
+        taskInfo:
+          name: print-op-4
+        inputs:
+          parameters:
+            msg:
+              runtimeValue:
+                constant: 'task name:'
+            value:
+              runtimeValue:
+                constant: '{{$.pipeline_task_name}}'
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op-4
+      print-op-5:
+        taskInfo:
+          name: print-op-5
+        inputs:
+          parameters:
+            msg:
+              runtimeValue:
+                constant: 'task id:'
+            value:
+              runtimeValue:
+                constant: '{{$.pipeline_task_uuid}}'
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-print-op-5

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_resource_spec.json

@@ -1,171 +0,0 @@
-{
-  "components": {
-    "comp-ingestion": {
-      "executorLabel": "exec-ingestion",
-      "inputDefinitions": {
-        "parameters": {
-          "input_location": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "examples": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    },
-    "comp-trainer": {
-      "executorLabel": "exec-trainer",
-      "inputDefinitions": {
-        "artifacts": {
-          "examples": {
-            "artifactType": {
-              "schemaTitle": "system.Dataset",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        },
-        "parameters": {
-          "n_epochs": {
-            "parameterType": "NUMBER_INTEGER"
-          },
-          "optimizer": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "model": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-ingestion": {
-        "container": {
-          "args": [
-            "--input-location",
-            "{{$.inputs.parameters['input_location']}}",
-            "--output-examples",
-            "{{$.outputs.artifacts['examples'].uri}}"
-          ],
-          "image": "gcr.io/my-project/my-ingestor"
-        }
-      },
-      "exec-trainer": {
-        "container": {
-          "args": [
-            "--input-examples",
-            "{{$.inputs.artifacts['examples'].uri}}",
-            "--optimizer",
-            "{{$.inputs.parameters['optimizer']}}",
-            "--n_epochs",
-            "{{$.inputs.parameters['n_epochs']}}",
-            "--output-model",
-            "{{$.outputs.artifacts['model'].uri}}"
-          ],
-          "image": "gcr.io/my-project/my-fancy-trainer",
-          "resources": {
-            "accelerator": {
-              "count": "1",
-              "type": "tpu-v3"
-            },
-            "cpuLimit": 4.0,
-            "memoryLimit": 15.032385536
-          }
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "two-step-pipeline-with-resource-spec"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "ingestion": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-ingestion"
-          },
-          "inputs": {
-            "parameters": {
-              "input_location": {
-                "componentInputParameter": "input_location"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "ingestion"
-          }
-        },
-        "trainer": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-trainer"
-          },
-          "dependentTasks": [
-            "ingestion"
-          ],
-          "inputs": {
-            "artifacts": {
-              "examples": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "examples",
-                  "producerTask": "ingestion"
-                }
-              }
-            },
-            "parameters": {
-              "n_epochs": {
-                "componentInputParameter": "n_epochs"
-              },
-              "optimizer": {
-                "componentInputParameter": "optimizer"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "trainer"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "input_location": {
-          "defaultValue": "gs://test-bucket/pipeline_root",
-          "parameterType": "STRING"
-        },
-        "n_epochs": {
-          "defaultValue": 200.0,
-          "parameterType": "NUMBER_INTEGER"
-        },
-        "optimizer": {
-          "defaultValue": "sgd",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_resource_spec.py

@@ -45,4 +45,4 @@ def my_pipeline(input_location: str = 'gs://test-bucket/pipeline_root',
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_resource_spec.yaml

@@ -0,0 +1,110 @@
+pipelineInfo:
+  name: two-step-pipeline-with-resource-spec
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-ingestion:
+      container:
+        image: gcr.io/my-project/my-ingestor
+        args:
+        - --input-location
+        - '{{$.inputs.parameters[''input_location'']}}'
+        - --output-examples
+        - '{{$.outputs.artifacts[''examples''].uri}}'
+    exec-trainer:
+      container:
+        image: gcr.io/my-project/my-fancy-trainer
+        args:
+        - --input-examples
+        - '{{$.inputs.artifacts[''examples''].uri}}'
+        - --optimizer
+        - '{{$.inputs.parameters[''optimizer'']}}'
+        - --n_epochs
+        - '{{$.inputs.parameters[''n_epochs'']}}'
+        - --output-model
+        - '{{$.outputs.artifacts[''model''].uri}}'
+        resources:
+          cpuLimit: 4.0
+          memoryLimit: 15.032385536
+          accelerator:
+            type: tpu-v3
+            count: '1'
+components:
+  comp-ingestion:
+    inputDefinitions:
+      parameters:
+        input_location:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        examples:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+    executorLabel: exec-ingestion
+  comp-trainer:
+    inputDefinitions:
+      artifacts:
+        examples:
+          artifactType:
+            schemaTitle: system.Dataset
+            schemaVersion: 0.0.1
+      parameters:
+        optimizer:
+          parameterType: STRING
+        n_epochs:
+          parameterType: NUMBER_INTEGER
+    outputDefinitions:
+      artifacts:
+        model:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+    executorLabel: exec-trainer
+root:
+  inputDefinitions:
+    parameters:
+      input_location:
+        parameterType: STRING
+        defaultValue: gs://test-bucket/pipeline_root
+      optimizer:
+        parameterType: STRING
+        defaultValue: sgd
+      n_epochs:
+        parameterType: NUMBER_INTEGER
+        defaultValue: 200.0
+  dag:
+    tasks:
+      ingestion:
+        taskInfo:
+          name: ingestion
+        inputs:
+          parameters:
+            input_location:
+              componentInputParameter: input_location
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-ingestion
+      trainer:
+        taskInfo:
+          name: trainer
+        inputs:
+          parameters:
+            optimizer:
+              componentInputParameter: optimizer
+            n_epochs:
+              componentInputParameter: n_epochs
+          artifacts:
+            examples:
+              taskOutputArtifact:
+                producerTask: ingestion
+                outputArtifactKey: examples
+        dependentTasks:
+        - ingestion
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-trainer
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_reused_component.json

@@ -1,210 +0,0 @@
-{
-  "components": {
-    "comp-add": {
-      "executorLabel": "exec-add",
-      "inputDefinitions": {
-        "parameters": {
-          "op2": {
-            "parameterType": "NUMBER_INTEGER"
-          },
-          "op_1": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "parameters": {
-          "sum": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-add-2": {
-      "executorLabel": "exec-add-2",
-      "inputDefinitions": {
-        "parameters": {
-          "op2": {
-            "parameterType": "NUMBER_INTEGER"
-          },
-          "op_1": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "parameters": {
-          "sum": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-add-3": {
-      "executorLabel": "exec-add-3",
-      "inputDefinitions": {
-        "parameters": {
-          "op2": {
-            "parameterType": "NUMBER_INTEGER"
-          },
-          "op_1": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "parameters": {
-          "sum": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-add": {
-        "container": {
-          "command": [
-            "sh",
-            "-c",
-            "set -e -x\necho \"$(($0+$1))\" | gsutil cp - \"$2\"\n",
-            "{{$.inputs.parameters['op_1']}}",
-            "{{$.inputs.parameters['op2']}}",
-            "{{$.outputs.parameters['sum'].output_file}}"
-          ],
-          "image": "google/cloud-sdk:latest"
-        }
-      },
-      "exec-add-2": {
-        "container": {
-          "command": [
-            "sh",
-            "-c",
-            "set -e -x\necho \"$(($0+$1))\" | gsutil cp - \"$2\"\n",
-            "{{$.inputs.parameters['op_1']}}",
-            "{{$.inputs.parameters['op2']}}",
-            "{{$.outputs.parameters['sum'].output_file}}"
-          ],
-          "image": "google/cloud-sdk:latest"
-        }
-      },
-      "exec-add-3": {
-        "container": {
-          "command": [
-            "sh",
-            "-c",
-            "set -e -x\necho \"$(($0+$1))\" | gsutil cp - \"$2\"\n",
-            "{{$.inputs.parameters['op_1']}}",
-            "{{$.inputs.parameters['op2']}}",
-            "{{$.outputs.parameters['sum'].output_file}}"
-          ],
-          "image": "google/cloud-sdk:latest"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "add-pipeline"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "add": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-add"
-          },
-          "inputs": {
-            "parameters": {
-              "op2": {
-                "runtimeValue": {
-                  "constant": 3.0
-                }
-              },
-              "op_1": {
-                "componentInputParameter": "a"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "add"
-          }
-        },
-        "add-2": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-add-2"
-          },
-          "dependentTasks": [
-            "add"
-          ],
-          "inputs": {
-            "parameters": {
-              "op2": {
-                "componentInputParameter": "b"
-              },
-              "op_1": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "sum",
-                  "producerTask": "add"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "add-2"
-          }
-        },
-        "add-3": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-add-3"
-          },
-          "dependentTasks": [
-            "add-2"
-          ],
-          "inputs": {
-            "parameters": {
-              "op2": {
-                "runtimeValue": {
-                  "constant": 7.0
-                }
-              },
-              "op_1": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "sum",
-                  "producerTask": "add-2"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "add-3"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "a": {
-          "defaultValue": 2.0,
-          "parameterType": "NUMBER_INTEGER"
-        },
-        "b": {
-          "defaultValue": 5.0,
-          "parameterType": "NUMBER_INTEGER"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_reused_component.py

@@ -36,4 +36,4 @@ def my_pipeline(
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_reused_component.yaml

@@ -0,0 +1,146 @@
+pipelineInfo:
+  name: add-pipeline
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-add:
+      container:
+        image: google/cloud-sdk:latest
+        command:
+        - sh
+        - -c
+        - 'set -e -x
+
+          echo "$(($0+$1))" | gsutil cp - "$2"
+
+          '
+        - '{{$.inputs.parameters[''op_1'']}}'
+        - '{{$.inputs.parameters[''op2'']}}'
+        - '{{$.outputs.parameters[''sum''].output_file}}'
+    exec-add-2:
+      container:
+        image: google/cloud-sdk:latest
+        command:
+        - sh
+        - -c
+        - 'set -e -x
+
+          echo "$(($0+$1))" | gsutil cp - "$2"
+
+          '
+        - '{{$.inputs.parameters[''op_1'']}}'
+        - '{{$.inputs.parameters[''op2'']}}'
+        - '{{$.outputs.parameters[''sum''].output_file}}'
+    exec-add-3:
+      container:
+        image: google/cloud-sdk:latest
+        command:
+        - sh
+        - -c
+        - 'set -e -x
+
+          echo "$(($0+$1))" | gsutil cp - "$2"
+
+          '
+        - '{{$.inputs.parameters[''op_1'']}}'
+        - '{{$.inputs.parameters[''op2'']}}'
+        - '{{$.outputs.parameters[''sum''].output_file}}'
+components:
+  comp-add:
+    inputDefinitions:
+      parameters:
+        op_1:
+          parameterType: NUMBER_INTEGER
+        op2:
+          parameterType: NUMBER_INTEGER
+    outputDefinitions:
+      parameters:
+        sum:
+          parameterType: NUMBER_INTEGER
+    executorLabel: exec-add
+  comp-add-2:
+    inputDefinitions:
+      parameters:
+        op_1:
+          parameterType: NUMBER_INTEGER
+        op2:
+          parameterType: NUMBER_INTEGER
+    outputDefinitions:
+      parameters:
+        sum:
+          parameterType: NUMBER_INTEGER
+    executorLabel: exec-add-2
+  comp-add-3:
+    inputDefinitions:
+      parameters:
+        op_1:
+          parameterType: NUMBER_INTEGER
+        op2:
+          parameterType: NUMBER_INTEGER
+    outputDefinitions:
+      parameters:
+        sum:
+          parameterType: NUMBER_INTEGER
+    executorLabel: exec-add-3
+root:
+  inputDefinitions:
+    parameters:
+      a:
+        parameterType: NUMBER_INTEGER
+        defaultValue: 2.0
+      b:
+        parameterType: NUMBER_INTEGER
+        defaultValue: 5.0
+  dag:
+    tasks:
+      add:
+        taskInfo:
+          name: add
+        inputs:
+          parameters:
+            op_1:
+              componentInputParameter: a
+            op2:
+              runtimeValue:
+                constant: 3.0
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-add
+      add-2:
+        taskInfo:
+          name: add-2
+        inputs:
+          parameters:
+            op_1:
+              taskOutputParameter:
+                producerTask: add
+                outputParameterKey: sum
+            op2:
+              componentInputParameter: b
+        dependentTasks:
+        - add
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-add-2
+      add-3:
+        taskInfo:
+          name: add-3
+        inputs:
+          parameters:
+            op_1:
+              taskOutputParameter:
+                producerTask: add-2
+                outputParameterKey: sum
+            op2:
+              runtimeValue:
+                constant: 7.0
+        dependentTasks:
+        - add-2
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-add-3
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_task_final_status.json

@@ -1,215 +0,0 @@
-{
-  "components": {
-    "comp-exit-handler-1": {
-      "dag": {
-        "tasks": {
-          "fail-op": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-fail-op"
-            },
-            "inputs": {
-              "parameters": {
-                "message": {
-                  "runtimeValue": {
-                    "constant": "Task failed."
-                  }
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "fail-op"
-            }
-          },
-          "print-op": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op"
-            },
-            "inputs": {
-              "parameters": {
-                "message": {
-                  "componentInputParameter": "pipelinechannel--message"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--message": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-exit-op": {
-      "executorLabel": "exec-exit-op",
-      "inputDefinitions": {
-        "parameters": {
-          "status": {
-            "parameterType": "STRUCT"
-          },
-          "user_input": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-fail-op": {
-      "executorLabel": "exec-fail-op",
-      "inputDefinitions": {
-        "parameters": {
-          "message": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op": {
-      "executorLabel": "exec-print-op",
-      "inputDefinitions": {
-        "parameters": {
-          "message": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-exit-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "exit_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'git+https://github.com/kubeflow/pipelines.git@master#subdirectory=sdk/python' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef exit_op(user_input: str, status: PipelineTaskFinalStatus):\n    \"\"\"Checks pipeline run status.\"\"\"\n    print('Pipeline status: ', status.state)\n    print('Job resource name: ', status.pipeline_job_resource_name)\n    print('Error code: ', status.error_code)\n    print('Error message: ', status.error_message)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-fail-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "fail_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef fail_op(message: str):\n    \"\"\"Fails.\"\"\"\n    import sys\n    print(message)\n    sys.exit(1)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "print_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(message: str):\n    \"\"\"Prints a message.\"\"\"\n    print(message)\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-task-final-status"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "exit-handler-1": {
-          "componentRef": {
-            "name": "comp-exit-handler-1"
-          },
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--message": {
-                "componentInputParameter": "message"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "my-pipeline"
-          }
-        },
-        "exit-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-exit-op"
-          },
-          "dependentTasks": [
-            "exit-handler-1"
-          ],
-          "inputs": {
-            "parameters": {
-              "status": {
-                "taskFinalStatus": {
-                  "producerTask": "exit-handler-1"
-                }
-              },
-              "user_input": {
-                "componentInputParameter": "message"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "exit-op"
-          },
-          "triggerPolicy": {
-            "strategy": "ALL_UPSTREAM_TASKS_COMPLETED"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "message": {
-          "defaultValue": "Hello World!",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_task_final_status.py

@@ -58,4 +58,4 @@ def my_pipeline(message: str = 'Hello World!'):
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_task_final_status.yaml

@@ -0,0 +1,176 @@
+pipelineInfo:
+  name: pipeline-with-task-final-status
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-print-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef print_op(message: str):\n    \"\"\"Prints a message.\"\"\"\n\
+          \    print(message)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - print_op
+    exec-fail-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef fail_op(message: str):\n    \"\"\"Fails.\"\"\"\n    import sys\n\
+          \    print(message)\n    sys.exit(1)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - fail_op
+    exec-exit-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'git+https://github.com/kubeflow/pipelines.git@master#subdirectory=sdk/python'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef exit_op(user_input: str, status: PipelineTaskFinalStatus):\n\
+          \    \"\"\"Checks pipeline run status.\"\"\"\n    print('Pipeline status:\
+          \ ', status.state)\n    print('Job resource name: ', status.pipeline_job_resource_name)\n\
+          \    print('Error code: ', status.error_code)\n    print('Error message:\
+          \ ', status.error_message)\n\n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - exit_op
+components:
+  comp-exit-handler-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--message:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-op:
+          taskInfo:
+            name: print-op
+          inputs:
+            parameters:
+              message:
+                componentInputParameter: pipelinechannel--message
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op
+        fail-op:
+          taskInfo:
+            name: fail-op
+          inputs:
+            parameters:
+              message:
+                runtimeValue:
+                  constant: Task failed.
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-fail-op
+  comp-print-op:
+    inputDefinitions:
+      parameters:
+        message:
+          parameterType: STRING
+    executorLabel: exec-print-op
+  comp-fail-op:
+    inputDefinitions:
+      parameters:
+        message:
+          parameterType: STRING
+    executorLabel: exec-fail-op
+  comp-exit-op:
+    inputDefinitions:
+      parameters:
+        user_input:
+          parameterType: STRING
+        status:
+          parameterType: STRUCT
+    executorLabel: exec-exit-op
+root:
+  inputDefinitions:
+    parameters:
+      message:
+        parameterType: STRING
+        defaultValue: Hello World!
+  dag:
+    tasks:
+      exit-handler-1:
+        taskInfo:
+          name: my-pipeline
+        inputs:
+          parameters:
+            pipelinechannel--message:
+              componentInputParameter: message
+        componentRef:
+          name: comp-exit-handler-1
+      exit-op:
+        taskInfo:
+          name: exit-op
+        inputs:
+          parameters:
+            user_input:
+              componentInputParameter: message
+            status:
+              taskFinalStatus:
+                producerTask: exit-handler-1
+        dependentTasks:
+        - exit-handler-1
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-exit-op
+        triggerPolicy:
+          strategy: ALL_UPSTREAM_TASKS_COMPLETED

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_task_final_status_yaml.json

@@ -1,146 +0,0 @@
-{
-  "components": {
-    "comp-exit-handler-1": {
-      "dag": {
-        "tasks": {
-          "print-op": {
-            "cachingOptions": {
-              "enableCache": true
-            },
-            "componentRef": {
-              "name": "comp-print-op"
-            },
-            "inputs": {
-              "parameters": {
-                "message": {
-                  "componentInputParameter": "pipelinechannel--message"
-                }
-              }
-            },
-            "taskInfo": {
-              "name": "print-op"
-            }
-          }
-        }
-      },
-      "inputDefinitions": {
-        "parameters": {
-          "pipelinechannel--message": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-exit-op": {
-      "executorLabel": "exec-exit-op",
-      "inputDefinitions": {
-        "parameters": {
-          "status": {
-            "parameterType": "STRUCT"
-          },
-          "user_input": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    },
-    "comp-print-op": {
-      "executorLabel": "exec-print-op",
-      "inputDefinitions": {
-        "parameters": {
-          "message": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-exit-op": {
-        "container": {
-          "command": [
-            "echo",
-            "user input:",
-            "{{$.inputs.parameters['user_input']}}",
-            "pipeline status:",
-            "{{$.inputs.parameters['status']}}"
-          ],
-          "image": "python:3.7"
-        }
-      },
-      "exec-print-op": {
-        "container": {
-          "command": [
-            "echo",
-            "{{$.inputs.parameters['message']}}"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-task-final-status-yaml"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "exit-handler-1": {
-          "componentRef": {
-            "name": "comp-exit-handler-1"
-          },
-          "inputs": {
-            "parameters": {
-              "pipelinechannel--message": {
-                "componentInputParameter": "message"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "my-pipeline"
-          }
-        },
-        "exit-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-exit-op"
-          },
-          "dependentTasks": [
-            "exit-handler-1"
-          ],
-          "inputs": {
-            "parameters": {
-              "status": {
-                "taskFinalStatus": {
-                  "producerTask": "exit-handler-1"
-                }
-              },
-              "user_input": {
-                "componentInputParameter": "message"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "exit-op"
-          },
-          "triggerPolicy": {
-            "strategy": "ALL_UPSTREAM_TASKS_COMPLETED"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "message": {
-          "defaultValue": "Hello World!",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_task_final_status_yaml.py

@@ -58,4 +58,4 @@ def my_pipeline(message: str = 'Hello World!'):
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_task_final_status_yaml.yaml

@@ -0,0 +1,89 @@
+pipelineInfo:
+  name: pipeline-with-task-final-status-yaml
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-print-op:
+      container:
+        image: python:3.7
+        command:
+        - echo
+        - '{{$.inputs.parameters[''message'']}}'
+    exec-exit-op:
+      container:
+        image: python:3.7
+        command:
+        - echo
+        - 'user input:'
+        - '{{$.inputs.parameters[''user_input'']}}'
+        - 'pipeline status:'
+        - '{{$.inputs.parameters[''status'']}}'
+components:
+  comp-exit-handler-1:
+    inputDefinitions:
+      parameters:
+        pipelinechannel--message:
+          parameterType: STRING
+    dag:
+      tasks:
+        print-op:
+          taskInfo:
+            name: print-op
+          inputs:
+            parameters:
+              message:
+                componentInputParameter: pipelinechannel--message
+          cachingOptions:
+            enableCache: true
+          componentRef:
+            name: comp-print-op
+  comp-print-op:
+    inputDefinitions:
+      parameters:
+        message:
+          parameterType: STRING
+    executorLabel: exec-print-op
+  comp-exit-op:
+    inputDefinitions:
+      parameters:
+        user_input:
+          parameterType: STRING
+        status:
+          parameterType: STRUCT
+    executorLabel: exec-exit-op
+root:
+  inputDefinitions:
+    parameters:
+      message:
+        parameterType: STRING
+        defaultValue: Hello World!
+  dag:
+    tasks:
+      exit-handler-1:
+        taskInfo:
+          name: my-pipeline
+        inputs:
+          parameters:
+            pipelinechannel--message:
+              componentInputParameter: message
+        componentRef:
+          name: comp-exit-handler-1
+      exit-op:
+        taskInfo:
+          name: exit-op
+        inputs:
+          parameters:
+            user_input:
+              componentInputParameter: message
+            status:
+              taskFinalStatus:
+                producerTask: exit-handler-1
+        dependentTasks:
+        - exit-handler-1
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-exit-op
+        triggerPolicy:
+          strategy: ALL_UPSTREAM_TASKS_COMPLETED

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_various_io_types.json

@@ -1,307 +0,0 @@
-{
-  "components": {
-    "comp-downstream": {
-      "executorLabel": "exec-downstream",
-      "inputDefinitions": {
-        "artifacts": {
-          "input_b": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "input_c": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "input_d": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "input_e": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "input_f": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "input_g": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "input_h": {
-            "artifactType": {
-              "schemaTitle": "system.HTML",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "input_i": {
-            "artifactType": {
-              "schemaTitle": "google.BQMLModel",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        },
-        "parameters": {
-          "input_a": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    },
-    "comp-upstream": {
-      "executorLabel": "exec-upstream",
-      "inputDefinitions": {
-        "parameters": {
-          "input_1": {
-            "parameterType": "STRING"
-          },
-          "input_2": {
-            "parameterType": "NUMBER_DOUBLE"
-          },
-          "input_3": {
-            "parameterType": "STRING"
-          },
-          "input_4": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "output_2": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "output_3": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "output_4": {
-            "artifactType": {
-              "schemaTitle": "system.Model",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "output_5": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "output_6": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "output_7": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "output_8": {
-            "artifactType": {
-              "schemaTitle": "system.HTML",
-              "schemaVersion": "0.0.1"
-            }
-          },
-          "output_9": {
-            "artifactType": {
-              "schemaTitle": "google.BQMLModel",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        },
-        "parameters": {
-          "output_1": {
-            "parameterType": "NUMBER_INTEGER"
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-downstream": {
-        "container": {
-          "args": [
-            "{{$.inputs.parameters['input_a']}}",
-            "{{$.inputs.artifacts['input_b'].uri}}",
-            "{{$.inputs.artifacts['input_c'].path}}",
-            "{{$.inputs.artifacts['input_d'].uri}}",
-            "{{$.inputs.artifacts['input_e'].uri}}",
-            "{{$.inputs.artifacts['input_f'].path}}",
-            "{{$.inputs.artifacts['input_g'].path}}",
-            "{{$.inputs.artifacts['input_h'].path}}"
-          ],
-          "image": "gcr.io/image"
-        }
-      },
-      "exec-upstream": {
-        "container": {
-          "args": [
-            "{{$.inputs.parameters['input_1']}}",
-            "{{$.inputs.parameters['input_2']}}",
-            "{{$.inputs.parameters['input_3']}}",
-            "{{$.inputs.parameters['input_4']}}",
-            "{{$.outputs.parameters['output_1'].output_file}}",
-            "{{$.outputs.artifacts['output_2'].uri}}",
-            "{{$.outputs.artifacts['output_3'].path}}",
-            "{{$.outputs.artifacts['output_4'].uri}}",
-            "{{$.outputs.artifacts['output_5'].uri}}",
-            "{{$.outputs.artifacts['output_6'].path}}",
-            "{{$.outputs.artifacts['output_7'].path}}",
-            "{{$.outputs.artifacts['output_8'].path}}"
-          ],
-          "image": "gcr.io/image"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "pipeline-with-various-types"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "downstream": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-downstream"
-          },
-          "dependentTasks": [
-            "upstream"
-          ],
-          "inputs": {
-            "artifacts": {
-              "input_b": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_2",
-                  "producerTask": "upstream"
-                }
-              },
-              "input_c": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_3",
-                  "producerTask": "upstream"
-                }
-              },
-              "input_d": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_4",
-                  "producerTask": "upstream"
-                }
-              },
-              "input_e": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_5",
-                  "producerTask": "upstream"
-                }
-              },
-              "input_f": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_6",
-                  "producerTask": "upstream"
-                }
-              },
-              "input_g": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_7",
-                  "producerTask": "upstream"
-                }
-              },
-              "input_h": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_8",
-                  "producerTask": "upstream"
-                }
-              },
-              "input_i": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_9",
-                  "producerTask": "upstream"
-                }
-              }
-            },
-            "parameters": {
-              "input_a": {
-                "taskOutputParameter": {
-                  "outputParameterKey": "output_1",
-                  "producerTask": "upstream"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "downstream"
-          }
-        },
-        "upstream": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-upstream"
-          },
-          "inputs": {
-            "parameters": {
-              "input_1": {
-                "componentInputParameter": "input1"
-              },
-              "input_2": {
-                "runtimeValue": {
-                  "constant": 3.1415926
-                }
-              },
-              "input_3": {
-                "componentInputParameter": "input3"
-              },
-              "input_4": {
-                "componentInputParameter": "input4"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "upstream"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "input1": {
-          "parameterType": "STRING"
-        },
-        "input3": {
-          "parameterType": "STRING"
-        },
-        "input4": {
-          "defaultValue": "",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_various_io_types.py

@@ -104,4 +104,4 @@ def my_pipeline(input1: str, input3: str, input4: str = ''):
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_various_io_types.yaml

@@ -0,0 +1,202 @@
+pipelineInfo:
+  name: pipeline-with-various-types
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-upstream:
+      container:
+        image: gcr.io/image
+        args:
+        - '{{$.inputs.parameters[''input_1'']}}'
+        - '{{$.inputs.parameters[''input_2'']}}'
+        - '{{$.inputs.parameters[''input_3'']}}'
+        - '{{$.inputs.parameters[''input_4'']}}'
+        - '{{$.outputs.parameters[''output_1''].output_file}}'
+        - '{{$.outputs.artifacts[''output_2''].uri}}'
+        - '{{$.outputs.artifacts[''output_3''].path}}'
+        - '{{$.outputs.artifacts[''output_4''].uri}}'
+        - '{{$.outputs.artifacts[''output_5''].uri}}'
+        - '{{$.outputs.artifacts[''output_6''].path}}'
+        - '{{$.outputs.artifacts[''output_7''].path}}'
+        - '{{$.outputs.artifacts[''output_8''].path}}'
+    exec-downstream:
+      container:
+        image: gcr.io/image
+        args:
+        - '{{$.inputs.parameters[''input_a'']}}'
+        - '{{$.inputs.artifacts[''input_b''].uri}}'
+        - '{{$.inputs.artifacts[''input_c''].path}}'
+        - '{{$.inputs.artifacts[''input_d''].uri}}'
+        - '{{$.inputs.artifacts[''input_e''].uri}}'
+        - '{{$.inputs.artifacts[''input_f''].path}}'
+        - '{{$.inputs.artifacts[''input_g''].path}}'
+        - '{{$.inputs.artifacts[''input_h''].path}}'
+components:
+  comp-upstream:
+    inputDefinitions:
+      parameters:
+        input_1:
+          parameterType: STRING
+        input_2:
+          parameterType: NUMBER_DOUBLE
+        input_3:
+          parameterType: STRING
+        input_4:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        output_2:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+        output_3:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        output_4:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+        output_5:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        output_6:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        output_7:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        output_8:
+          artifactType:
+            schemaTitle: system.HTML
+            schemaVersion: 0.0.1
+        output_9:
+          artifactType:
+            schemaTitle: google.BQMLModel
+            schemaVersion: 0.0.1
+      parameters:
+        output_1:
+          parameterType: NUMBER_INTEGER
+    executorLabel: exec-upstream
+  comp-downstream:
+    inputDefinitions:
+      artifacts:
+        input_b:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+        input_c:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        input_d:
+          artifactType:
+            schemaTitle: system.Model
+            schemaVersion: 0.0.1
+        input_e:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        input_f:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        input_g:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        input_h:
+          artifactType:
+            schemaTitle: system.HTML
+            schemaVersion: 0.0.1
+        input_i:
+          artifactType:
+            schemaTitle: google.BQMLModel
+            schemaVersion: 0.0.1
+      parameters:
+        input_a:
+          parameterType: NUMBER_INTEGER
+    executorLabel: exec-downstream
+root:
+  inputDefinitions:
+    parameters:
+      input1:
+        parameterType: STRING
+      input3:
+        parameterType: STRING
+      input4:
+        parameterType: STRING
+        defaultValue: ''
+  dag:
+    tasks:
+      upstream:
+        taskInfo:
+          name: upstream
+        inputs:
+          parameters:
+            input_1:
+              componentInputParameter: input1
+            input_2:
+              runtimeValue:
+                constant: 3.1415926
+            input_3:
+              componentInputParameter: input3
+            input_4:
+              componentInputParameter: input4
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-upstream
+      downstream:
+        taskInfo:
+          name: downstream
+        inputs:
+          parameters:
+            input_a:
+              taskOutputParameter:
+                producerTask: upstream
+                outputParameterKey: output_1
+          artifacts:
+            input_b:
+              taskOutputArtifact:
+                producerTask: upstream
+                outputArtifactKey: output_2
+            input_c:
+              taskOutputArtifact:
+                producerTask: upstream
+                outputArtifactKey: output_3
+            input_d:
+              taskOutputArtifact:
+                producerTask: upstream
+                outputArtifactKey: output_4
+            input_e:
+              taskOutputArtifact:
+                producerTask: upstream
+                outputArtifactKey: output_5
+            input_f:
+              taskOutputArtifact:
+                producerTask: upstream
+                outputArtifactKey: output_6
+            input_g:
+              taskOutputArtifact:
+                producerTask: upstream
+                outputArtifactKey: output_7
+            input_h:
+              taskOutputArtifact:
+                producerTask: upstream
+                outputArtifactKey: output_8
+            input_i:
+              taskOutputArtifact:
+                producerTask: upstream
+                outputArtifactKey: output_9
+        dependentTasks:
+        - upstream
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-downstream
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/two_step_pipeline.json

@@ -1,126 +0,0 @@
-{
-  "components": {
-    "comp-read-from-gcs": {
-      "executorLabel": "exec-read-from-gcs",
-      "inputDefinitions": {
-        "artifacts": {
-          "input_gcs_path": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    },
-    "comp-write-to-gcs": {
-      "executorLabel": "exec-write-to-gcs",
-      "inputDefinitions": {
-        "parameters": {
-          "text": {
-            "parameterType": "STRING"
-          }
-        }
-      },
-      "outputDefinitions": {
-        "artifacts": {
-          "output_gcs_path": {
-            "artifactType": {
-              "schemaTitle": "system.Artifact",
-              "schemaVersion": "0.0.1"
-            }
-          }
-        }
-      }
-    }
-  },
-  "defaultPipelineRoot": "dummy_root",
-  "deploymentSpec": {
-    "executors": {
-      "exec-read-from-gcs": {
-        "container": {
-          "command": [
-            "sh",
-            "-c",
-            "set -e -x\ngsutil cat \"$0\"\n",
-            "{{$.inputs.artifacts['input_gcs_path'].uri}}"
-          ],
-          "image": "google/cloud-sdk:slim"
-        }
-      },
-      "exec-write-to-gcs": {
-        "container": {
-          "command": [
-            "sh",
-            "-c",
-            "set -e -x\necho \"$0\" | gsutil cp - \"$1\"\n",
-            "{{$.inputs.parameters['text']}}",
-            "{{$.outputs.artifacts['output_gcs_path'].uri}}"
-          ],
-          "image": "google/cloud-sdk:slim"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "simple-two-step-pipeline"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "read-from-gcs": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-read-from-gcs"
-          },
-          "dependentTasks": [
-            "write-to-gcs"
-          ],
-          "inputs": {
-            "artifacts": {
-              "input_gcs_path": {
-                "taskOutputArtifact": {
-                  "outputArtifactKey": "output_gcs_path",
-                  "producerTask": "write-to-gcs"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "Consumer"
-          }
-        },
-        "write-to-gcs": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-write-to-gcs"
-          },
-          "inputs": {
-            "parameters": {
-              "text": {
-                "componentInputParameter": "text"
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "Producer"
-          }
-        }
-      }
-    },
-    "inputDefinitions": {
-      "parameters": {
-        "text": {
-          "defaultValue": "Hello KFP!",
-          "parameterType": "STRING"
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/two_step_pipeline.py

@@ -66,4 +66,4 @@ if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=my_pipeline,
         pipeline_parameters={'text': 'Hello KFP!'},
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/two_step_pipeline.yaml

@@ -0,0 +1,87 @@
+pipelineInfo:
+  name: simple-two-step-pipeline
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-write-to-gcs:
+      container:
+        image: google/cloud-sdk:slim
+        command:
+        - sh
+        - -c
+        - 'set -e -x
+
+          echo "$0" | gsutil cp - "$1"
+
+          '
+        - '{{$.inputs.parameters[''text'']}}'
+        - '{{$.outputs.artifacts[''output_gcs_path''].uri}}'
+    exec-read-from-gcs:
+      container:
+        image: google/cloud-sdk:slim
+        command:
+        - sh
+        - -c
+        - 'set -e -x
+
+          gsutil cat "$0"
+
+          '
+        - '{{$.inputs.artifacts[''input_gcs_path''].uri}}'
+components:
+  comp-write-to-gcs:
+    inputDefinitions:
+      parameters:
+        text:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        output_gcs_path:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-write-to-gcs
+  comp-read-from-gcs:
+    inputDefinitions:
+      artifacts:
+        input_gcs_path:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-read-from-gcs
+root:
+  inputDefinitions:
+    parameters:
+      text:
+        parameterType: STRING
+        defaultValue: Hello KFP!
+  dag:
+    tasks:
+      write-to-gcs:
+        taskInfo:
+          name: Producer
+        inputs:
+          parameters:
+            text:
+              componentInputParameter: text
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-write-to-gcs
+      read-from-gcs:
+        taskInfo:
+          name: Consumer
+        inputs:
+          artifacts:
+            input_gcs_path:
+              taskOutputArtifact:
+                producerTask: write-to-gcs
+                outputArtifactKey: output_gcs_path
+        dependentTasks:
+        - write-to-gcs
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-read-from-gcs
+defaultPipelineRoot: dummy_root

sdk/python/kfp/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json

@@ -1,77 +0,0 @@
-{
-  "components": {
-    "comp-component-op": {
-      "executorLabel": "exec-component-op",
-      "inputDefinitions": {
-        "parameters": {
-          "input1": {
-            "parameterType": "STRING"
-          },
-          "input2": {
-            "parameterType": "STRING"
-          }
-        }
-      }
-    }
-  },
-  "deploymentSpec": {
-    "executors": {
-      "exec-component-op": {
-        "container": {
-          "args": [
-            "--executor_input",
-            "{{$}}",
-            "--function_to_execute",
-            "component_op"
-          ],
-          "command": [
-            "sh",
-            "-c",
-            "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.0' && \"$0\" \"$@\"\n",
-            "sh",
-            "-ec",
-            "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n",
-            "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_op(\n    input1: str = 'default value',\n    input2: Optional[str] = None,\n    input3: Optional[str] = None,\n):\n    print(f'input1: {input1}, type: {type(input1)}')\n    print(f'input2: {input2}, type: {type(input2)}')\n    print(f'input3: {input3}, type: {type(input3)}')\n\n"
-          ],
-          "image": "python:3.7"
-        }
-      }
-    }
-  },
-  "pipelineInfo": {
-    "name": "v2-component-optional-input"
-  },
-  "root": {
-    "dag": {
-      "tasks": {
-        "component-op": {
-          "cachingOptions": {
-            "enableCache": true
-          },
-          "componentRef": {
-            "name": "comp-component-op"
-          },
-          "inputs": {
-            "parameters": {
-              "input1": {
-                "runtimeValue": {
-                  "constant": "Hello"
-                }
-              },
-              "input2": {
-                "runtimeValue": {
-                  "constant": "World"
-                }
-              }
-            }
-          },
-          "taskInfo": {
-            "name": "component-op"
-          }
-        }
-      }
-    }
-  },
-  "schemaVersion": "2.1.0",
-  "sdkVersion": "kfp-1.8.11"
-}

sdk/python/kfp/compiler_cli_tests/test_data/v2_component_with_optional_inputs.py

@@ -38,4 +38,4 @@ def pipeline():
 
 if __name__ == '__main__':
     compiler.Compiler().compile(
-        pipeline_func=pipeline, package_path=__file__.replace('.py', '.json'))
+        pipeline_func=pipeline, package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/v2_component_with_optional_inputs.yaml

@@ -0,0 +1,63 @@
+pipelineInfo:
+  name: v2-component-optional-input
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-component-op:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet     --no-warn-script-location 'kfp==2.0.0-alpha.1'\
+          \ && \"$0\" \"$@\"\n"
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
+
+          printf "%s" "$0" > "$program_path/ephemeral_component.py"
+
+          python3 -m kfp.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
+
+          '
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+          \ *\n\ndef component_op(\n    input1: str = 'default value',\n    input2:\
+          \ Optional[str] = None,\n    input3: Optional[str] = None,\n):\n    print(f'input1:\
+          \ {input1}, type: {type(input1)}')\n    print(f'input2: {input2}, type:\
+          \ {type(input2)}')\n    print(f'input3: {input3}, type: {type(input3)}')\n\
+          \n"
+        args:
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - component_op
+components:
+  comp-component-op:
+    inputDefinitions:
+      parameters:
+        input1:
+          parameterType: STRING
+        input2:
+          parameterType: STRING
+    executorLabel: exec-component-op
+root:
+  dag:
+    tasks:
+      component-op:
+        taskInfo:
+          name: component-op
+        inputs:
+          parameters:
+            input1:
+              runtimeValue:
+                constant: Hello
+            input2:
+              runtimeValue:
+                constant: World
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-component-op

sdk/python/kfp/compiler_cli_tests/test_data/xgboost_sample_pipeline.py

@@ -92,4 +92,4 @@ def xgboost_pipeline():
 if __name__ == '__main__':
     compiler.Compiler().compile(
         pipeline_func=xgboost_pipeline,
-        package_path=__file__.replace('.py', '.json'))
+        package_path=__file__.replace('.py', '.yaml'))

sdk/python/kfp/compiler_cli_tests/test_data/xgboost_sample_pipeline.yaml

@@ -0,0 +1,834 @@
+pipelineInfo:
+  name: xgboost-sample-pipeline
+sdkVersion: kfp-2.0.0-alpha.1
+schemaVersion: 2.1.0
+deploymentSpec:
+  executors:
+    exec-chicago-taxi-trips-dataset:
+      container:
+        image: byrnedo/alpine-curl@sha256:548379d0a4a0c08b9e55d9d87a592b7d35d9ab3037f4936f5ccd09d0b625a342
+        command:
+        - sh
+        - -c
+        - "set -e -x -o pipefail\noutput_path=\"$0\"\nselect=\"$1\"\nwhere=\"$2\"\n\
+          limit=\"$3\"\nformat=\"$4\"\nmkdir -p \"$(dirname \"$output_path\")\"\n\
+          curl --get 'https://data.cityofchicago.org/resource/wrvz-psew.'\"${format}\"\
+          \ \\\n    --data-urlencode '$limit='\"${limit}\" \\\n    --data-urlencode\
+          \ '$where='\"${where}\" \\\n    --data-urlencode '$select='\"${select}\"\
+          \ \\\n    | tr -d '\"' > \"$output_path\"  # Removing unneeded quotes around\
+          \ all numbers\n"
+        - '{{$.outputs.artifacts[''table''].path}}'
+        - '{{$.inputs.parameters[''select'']}}'
+        - '{{$.inputs.parameters[''where'']}}'
+        - '{{$.inputs.parameters[''limit'']}}'
+    exec-xgboost-train:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+          'xgboost==1.1.1' 'pandas==1.0.5' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+          -m pip install --quiet --no-warn-script-location 'xgboost==1.1.1' 'pandas==1.0.5'
+          --user) && "$0" "$@"
+        - python3
+        - -u
+        - -c
+        - "def _make_parent_dirs_and_return_path(file_path: str):\n    import os\n\
+          \    os.makedirs(os.path.dirname(file_path), exist_ok=True)\n    return\
+          \ file_path\n\ndef xgboost_train(\n    training_data_path,  # Also supports\
+          \ LibSVM\n    model_path,\n    model_config_path,\n    starting_model_path\
+          \ = None,\n\n    label_column = 0,\n    num_iterations = 10,\n    booster_params\
+          \ = None,\n\n    # Booster parameters\n    objective = 'reg:squarederror',\n\
+          \    booster = 'gbtree',\n    learning_rate = 0.3,\n    min_split_loss =\
+          \ 0,\n    max_depth = 6,\n):\n    '''Train an XGBoost model.\n\n    Args:\n\
+          \        training_data_path: Path for the training data in CSV format.\n\
+          \        model_path: Output path for the trained model in binary XGBoost\
+          \ format.\n        model_config_path: Output path for the internal parameter\
+          \ configuration of Booster as a JSON string.\n        starting_model_path:\
+          \ Path for the existing trained model to start from.\n        label_column:\
+          \ Column containing the label data.\n        num_boost_rounds: Number of\
+          \ boosting iterations.\n        booster_params: Parameters for the booster.\
+          \ See https://xgboost.readthedocs.io/en/latest/parameter.html\n        objective:\
+          \ The learning task and the corresponding learning objective.\n        \
+          \    See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\n\
+          \            The most common values are:\n            \"reg:squarederror\"\
+          \ - Regression with squared loss (default).\n            \"reg:logistic\"\
+          \ - Logistic regression.\n            \"binary:logistic\" - Logistic regression\
+          \ for binary classification, output probability.\n            \"binary:logitraw\"\
+          \ - Logistic regression for binary classification, output score before logistic\
+          \ transformation\n            \"rank:pairwise\" - Use LambdaMART to perform\
+          \ pairwise ranking where the pairwise loss is minimized\n            \"\
+          rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized\
+          \ Discounted Cumulative Gain (NDCG) is maximized\n\n    Annotations:\n \
+          \       author: Alexey Volkov <alexey.volkov@ark-kun.com>\n    '''\n   \
+          \ import pandas\n    import xgboost\n\n    df = pandas.read_csv(\n     \
+          \   training_data_path,\n    )\n\n    training_data = xgboost.DMatrix(\n\
+          \        data=df.drop(columns=[df.columns[label_column]]),\n        label=df[df.columns[label_column]],\n\
+          \    )\n\n    booster_params = booster_params or {}\n    booster_params.setdefault('objective',\
+          \ objective)\n    booster_params.setdefault('booster', booster)\n    booster_params.setdefault('learning_rate',\
+          \ learning_rate)\n    booster_params.setdefault('min_split_loss', min_split_loss)\n\
+          \    booster_params.setdefault('max_depth', max_depth)\n\n    starting_model\
+          \ = None\n    if starting_model_path:\n        starting_model = xgboost.Booster(model_file=starting_model_path)\n\
+          \n    model = xgboost.train(\n        params=booster_params,\n        dtrain=training_data,\n\
+          \        num_boost_round=num_iterations,\n        xgb_model=starting_model\n\
+          \    )\n\n    # Saving the model in binary format\n    model.save_model(model_path)\n\
+          \n    model_config_str = model.save_config()\n    with open(model_config_path,\
+          \ 'w') as model_config_file:\n        model_config_file.write(model_config_str)\n\
+          \nimport json\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost\
+          \ train', description='Train an XGBoost model.\\n\\n    Args:\\n       \
+          \ training_data_path: Path for the training data in CSV format.\\n     \
+          \   model_path: Output path for the trained model in binary XGBoost format.\\\
+          n        model_config_path: Output path for the internal parameter configuration\
+          \ of Booster as a JSON string.\\n        starting_model_path: Path for the\
+          \ existing trained model to start from.\\n        label_column: Column containing\
+          \ the label data.\\n        num_boost_rounds: Number of boosting iterations.\\\
+          n        booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\\\
+          n        objective: The learning task and the corresponding learning objective.\\\
+          n            See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\\\
+          n            The most common values are:\\n            \"reg:squarederror\"\
+          \ - Regression with squared loss (default).\\n            \"reg:logistic\"\
+          \ - Logistic regression.\\n            \"binary:logistic\" - Logistic regression\
+          \ for binary classification, output probability.\\n            \"binary:logitraw\"\
+          \ - Logistic regression for binary classification, output score before logistic\
+          \ transformation\\n            \"rank:pairwise\" - Use LambdaMART to perform\
+          \ pairwise ranking where the pairwise loss is minimized\\n            \"\
+          rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized\
+          \ Discounted Cumulative Gain (NDCG) is maximized\\n\\n    Annotations:\\\
+          n        author: Alexey Volkov <alexey.volkov@ark-kun.com>')\n_parser.add_argument(\"\
+          --training-data\", dest=\"training_data_path\", type=str, required=True,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--starting-model\"\
+          , dest=\"starting_model_path\", type=str, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--label-column\", dest=\"label_column\", type=int,\
+          \ required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--num-iterations\"\
+          , dest=\"num_iterations\", type=int, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--booster-params\", dest=\"booster_params\", type=json.loads,\
+          \ required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--objective\"\
+          , dest=\"objective\", type=str, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--booster\", dest=\"booster\", type=str, required=False,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--learning-rate\",\
+          \ dest=\"learning_rate\", type=float, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--min-split-loss\", dest=\"min_split_loss\", type=float,\
+          \ required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--max-depth\"\
+          , dest=\"max_depth\", type=int, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--model\", dest=\"model_path\", type=_make_parent_dirs_and_return_path,\
+          \ required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model-config\"\
+          , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\
+          \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\
+          \n_outputs = xgboost_train(**_parsed_args)\n"
+        args:
+        - --training-data
+        - '{{$.inputs.artifacts[''training_data''].path}}'
+        - --label-column
+        - '{{$.inputs.parameters[''label_column'']}}'
+        - --num-iterations
+        - '{{$.inputs.parameters[''num_iterations'']}}'
+        - --objective
+        - '{{$.inputs.parameters[''objective'']}}'
+        - --model
+        - '{{$.outputs.artifacts[''model''].path}}'
+        - --model-config
+        - '{{$.outputs.artifacts[''model_config''].path}}'
+    exec-xgboost-predict:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+          'xgboost==1.1.1' 'pandas==1.0.5' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+          -m pip install --quiet --no-warn-script-location 'xgboost==1.1.1' 'pandas==1.0.5'
+          --user) && "$0" "$@"
+        - python3
+        - -u
+        - -c
+        - "def _make_parent_dirs_and_return_path(file_path: str):\n    import os\n\
+          \    os.makedirs(os.path.dirname(file_path), exist_ok=True)\n    return\
+          \ file_path\n\ndef xgboost_predict(\n    data_path,  # Also supports LibSVM\n\
+          \    model_path,\n    predictions_path,\n    label_column = None,\n):\n\
+          \    '''Make predictions using a trained XGBoost model.\n\n    Args:\n \
+          \       data_path: Path for the feature data in CSV format.\n        model_path:\
+          \ Path for the trained model in binary XGBoost format.\n        predictions_path:\
+          \ Output path for the predictions.\n        label_column: Column containing\
+          \ the label data.\n\n    Annotations:\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>\n\
+          \    '''\n    from pathlib import Path\n\n    import numpy\n    import pandas\n\
+          \    import xgboost\n\n    df = pandas.read_csv(\n        data_path,\n \
+          \   )\n\n    if label_column is not None:\n        df = df.drop(columns=[df.columns[label_column]])\n\
+          \n    testing_data = xgboost.DMatrix(\n        data=df,\n    )\n\n    model\
+          \ = xgboost.Booster(model_file=model_path)\n\n    predictions = model.predict(testing_data)\n\
+          \n    Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n\
+          \    numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser\
+          \ = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions\
+          \ using a trained XGBoost model.\\n\\n    Args:\\n        data_path: Path\
+          \ for the feature data in CSV format.\\n        model_path: Path for the\
+          \ trained model in binary XGBoost format.\\n        predictions_path: Output\
+          \ path for the predictions.\\n        label_column: Column containing the\
+          \ label data.\\n\\n    Annotations:\\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>')\n\
+          _parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"\
+          model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"\
+          --label-column\", dest=\"label_column\", type=int, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\
+          \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\
+          \n_outputs = xgboost_predict(**_parsed_args)\n"
+        args:
+        - --data
+        - '{{$.inputs.artifacts[''data''].path}}'
+        - --model
+        - '{{$.inputs.artifacts[''model''].path}}'
+        - --label-column
+        - '{{$.inputs.parameters[''label_column'']}}'
+        - --predictions
+        - '{{$.outputs.artifacts[''predictions''].path}}'
+    exec-convert-csv-to-apache-parquet:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+          'pyarrow==0.17.1' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install
+          --quiet --no-warn-script-location 'pyarrow==0.17.1' --user) && "$0" "$@"
+        - python3
+        - -u
+        - -c
+        - "def _make_parent_dirs_and_return_path(file_path: str):\n    import os\n\
+          \    os.makedirs(os.path.dirname(file_path), exist_ok=True)\n    return\
+          \ file_path\n\ndef convert_csv_to_apache_parquet(\n    data_path,\n    output_data_path,\n\
+          ):\n    '''Converts CSV table to Apache Parquet.\n\n    [Apache Parquet](https://parquet.apache.org/)\n\
+          \n    Annotations:\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>\n\
+          \    '''\n    from pyarrow import csv, parquet\n\n    table = csv.read_csv(data_path)\n\
+          \    parquet.write_table(table, output_data_path)\n\nimport argparse\n_parser\
+          \ = argparse.ArgumentParser(prog='Convert csv to apache parquet', description='Converts\
+          \ CSV table to Apache Parquet.\\n\\n    [Apache Parquet](https://parquet.apache.org/)\\\
+          n\\n    Annotations:\\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>')\n\
+          _parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--output-data\", dest=\"\
+          output_data_path\", type=_make_parent_dirs_and_return_path, required=True,\
+          \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\
+          _output_files = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = convert_csv_to_apache_parquet(**_parsed_args)\n\
+          \n_output_serializers = [\n\n]\n\nimport os\nfor idx, output_file in enumerate(_output_files):\n\
+          \    try:\n        os.makedirs(os.path.dirname(output_file))\n    except\
+          \ OSError:\n        pass\n    with open(output_file, 'w') as f:\n      \
+          \  f.write(_output_serializers[idx](_outputs[idx]))\n"
+        args:
+        - --data
+        - '{{$.inputs.artifacts[''data''].path}}'
+        - --output-data
+        - '{{$.outputs.artifacts[''output_data''].path}}'
+    exec-xgboost-train-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+          'xgboost==1.1.1' 'pandas==1.0.5' 'pyarrow==0.17.1' || PIP_DISABLE_PIP_VERSION_CHECK=1
+          python3 -m pip install --quiet --no-warn-script-location 'xgboost==1.1.1'
+          'pandas==1.0.5' 'pyarrow==0.17.1' --user) && "$0" "$@"
+        - python3
+        - -u
+        - -c
+        - "def _make_parent_dirs_and_return_path(file_path: str):\n    import os\n\
+          \    os.makedirs(os.path.dirname(file_path), exist_ok=True)\n    return\
+          \ file_path\n\ndef xgboost_train(\n    training_data_path,\n    model_path,\n\
+          \    model_config_path,\n    label_column_name,\n\n    starting_model_path\
+          \ = None,\n\n    num_iterations = 10,\n    booster_params = None,\n\n  \
+          \  # Booster parameters\n    objective = 'reg:squarederror',\n    booster\
+          \ = 'gbtree',\n    learning_rate = 0.3,\n    min_split_loss = 0,\n    max_depth\
+          \ = 6,\n):\n    '''Train an XGBoost model.\n\n    Args:\n        training_data_path:\
+          \ Path for the training data in Apache Parquet format.\n        model_path:\
+          \ Output path for the trained model in binary XGBoost format.\n        model_config_path:\
+          \ Output path for the internal parameter configuration of Booster as a JSON\
+          \ string.\n        starting_model_path: Path for the existing trained model\
+          \ to start from.\n        label_column_name: Name of the column containing\
+          \ the label data.\n        num_boost_rounds: Number of boosting iterations.\n\
+          \        booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\n\
+          \        objective: The learning task and the corresponding learning objective.\n\
+          \            See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\n\
+          \            The most common values are:\n            \"reg:squarederror\"\
+          \ - Regression with squared loss (default).\n            \"reg:logistic\"\
+          \ - Logistic regression.\n            \"binary:logistic\" - Logistic regression\
+          \ for binary classification, output probability.\n            \"binary:logitraw\"\
+          \ - Logistic regression for binary classification, output score before logistic\
+          \ transformation\n            \"rank:pairwise\" - Use LambdaMART to perform\
+          \ pairwise ranking where the pairwise loss is minimized\n            \"\
+          rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized\
+          \ Discounted Cumulative Gain (NDCG) is maximized\n\n    Annotations:\n \
+          \       author: Alexey Volkov <alexey.volkov@ark-kun.com>\n    '''\n   \
+          \ import pandas\n    import xgboost\n\n    # Loading data\n    df = pandas.read_parquet(training_data_path)\n\
+          \    training_data = xgboost.DMatrix(\n        data=df.drop(columns=[label_column_name]),\n\
+          \        label=df[[label_column_name]],\n    )\n    # Training\n    booster_params\
+          \ = booster_params or {}\n    booster_params.setdefault('objective', objective)\n\
+          \    booster_params.setdefault('booster', booster)\n    booster_params.setdefault('learning_rate',\
+          \ learning_rate)\n    booster_params.setdefault('min_split_loss', min_split_loss)\n\
+          \    booster_params.setdefault('max_depth', max_depth)\n\n    starting_model\
+          \ = None\n    if starting_model_path:\n        starting_model = xgboost.Booster(model_file=starting_model_path)\n\
+          \n    model = xgboost.train(\n        params=booster_params,\n        dtrain=training_data,\n\
+          \        num_boost_round=num_iterations,\n        xgb_model=starting_model\n\
+          \    )\n\n    # Saving the model in binary format\n    model.save_model(model_path)\n\
+          \n    model_config_str = model.save_config()\n    with open(model_config_path,\
+          \ 'w') as model_config_file:\n        model_config_file.write(model_config_str)\n\
+          \nimport json\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost\
+          \ train', description='Train an XGBoost model.\\n\\n    Args:\\n       \
+          \ training_data_path: Path for the training data in Apache Parquet format.\\\
+          n        model_path: Output path for the trained model in binary XGBoost\
+          \ format.\\n        model_config_path: Output path for the internal parameter\
+          \ configuration of Booster as a JSON string.\\n        starting_model_path:\
+          \ Path for the existing trained model to start from.\\n        label_column_name:\
+          \ Name of the column containing the label data.\\n        num_boost_rounds:\
+          \ Number of boosting iterations.\\n        booster_params: Parameters for\
+          \ the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\\\
+          n        objective: The learning task and the corresponding learning objective.\\\
+          n            See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\\\
+          n            The most common values are:\\n            \"reg:squarederror\"\
+          \ - Regression with squared loss (default).\\n            \"reg:logistic\"\
+          \ - Logistic regression.\\n            \"binary:logistic\" - Logistic regression\
+          \ for binary classification, output probability.\\n            \"binary:logitraw\"\
+          \ - Logistic regression for binary classification, output score before logistic\
+          \ transformation\\n            \"rank:pairwise\" - Use LambdaMART to perform\
+          \ pairwise ranking where the pairwise loss is minimized\\n            \"\
+          rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized\
+          \ Discounted Cumulative Gain (NDCG) is maximized\\n\\n    Annotations:\\\
+          n        author: Alexey Volkov <alexey.volkov@ark-kun.com>')\n_parser.add_argument(\"\
+          --training-data\", dest=\"training_data_path\", type=str, required=True,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column-name\"\
+          , dest=\"label_column_name\", type=str, required=True, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--starting-model\", dest=\"starting_model_path\"\
+          , type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"\
+          --num-iterations\", dest=\"num_iterations\", type=int, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--booster-params\", dest=\"booster_params\", type=json.loads,\
+          \ required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--objective\"\
+          , dest=\"objective\", type=str, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--booster\", dest=\"booster\", type=str, required=False,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--learning-rate\",\
+          \ dest=\"learning_rate\", type=float, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--min-split-loss\", dest=\"min_split_loss\", type=float,\
+          \ required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--max-depth\"\
+          , dest=\"max_depth\", type=int, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--model\", dest=\"model_path\", type=_make_parent_dirs_and_return_path,\
+          \ required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model-config\"\
+          , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\
+          \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\
+          \n_outputs = xgboost_train(**_parsed_args)\n"
+        args:
+        - --training-data
+        - '{{$.inputs.artifacts[''training_data''].path}}'
+        - --label-column-name
+        - '{{$.inputs.parameters[''label_column_name'']}}'
+        - --num-iterations
+        - '{{$.inputs.parameters[''num_iterations'']}}'
+        - --objective
+        - '{{$.inputs.parameters[''objective'']}}'
+        - --model
+        - '{{$.outputs.artifacts[''model''].path}}'
+        - --model-config
+        - '{{$.outputs.artifacts[''model_config''].path}}'
+    exec-xgboost-predict-2:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+          'xgboost==1.1.1' 'pandas==1.0.5' 'pyarrow==0.17.1' || PIP_DISABLE_PIP_VERSION_CHECK=1
+          python3 -m pip install --quiet --no-warn-script-location 'xgboost==1.1.1'
+          'pandas==1.0.5' 'pyarrow==0.17.1' --user) && "$0" "$@"
+        - python3
+        - -u
+        - -c
+        - "def _make_parent_dirs_and_return_path(file_path: str):\n    import os\n\
+          \    os.makedirs(os.path.dirname(file_path), exist_ok=True)\n    return\
+          \ file_path\n\ndef xgboost_predict(\n    data_path,\n    model_path,\n \
+          \   predictions_path,\n    label_column_name = None,\n):\n    '''Make predictions\
+          \ using a trained XGBoost model.\n\n    Args:\n        data_path: Path for\
+          \ the feature data in Apache Parquet format.\n        model_path: Path for\
+          \ the trained model in binary XGBoost format.\n        predictions_path:\
+          \ Output path for the predictions.\n        label_column_name: Optional.\
+          \ Name of the column containing the label data that is excluded during the\
+          \ prediction.\n\n    Annotations:\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>\n\
+          \    '''\n    from pathlib import Path\n\n    import numpy\n    import pandas\n\
+          \    import xgboost\n\n    # Loading data\n    df = pandas.read_parquet(data_path)\n\
+          \    if label_column_name:\n        df = df.drop(columns=[label_column_name])\n\
+          \n    evaluation_data = xgboost.DMatrix(\n        data=df,\n    )\n\n  \
+          \  # Training\n    model = xgboost.Booster(model_file=model_path)\n\n  \
+          \  predictions = model.predict(evaluation_data)\n\n    Path(predictions_path).parent.mkdir(parents=True,\
+          \ exist_ok=True)\n    numpy.savetxt(predictions_path, predictions)\n\nimport\
+          \ argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make\
+          \ predictions using a trained XGBoost model.\\n\\n    Args:\\n        data_path:\
+          \ Path for the feature data in Apache Parquet format.\\n        model_path:\
+          \ Path for the trained model in binary XGBoost format.\\n        predictions_path:\
+          \ Output path for the predictions.\\n        label_column_name: Optional.\
+          \ Name of the column containing the label data that is excluded during the\
+          \ prediction.\\n\\n    Annotations:\\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>')\n\
+          _parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"\
+          model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"\
+          --label-column-name\", dest=\"label_column_name\", type=str, required=False,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"\
+          predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\
+          \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\
+          \n_outputs = xgboost_predict(**_parsed_args)\n"
+        args:
+        - --data
+        - '{{$.inputs.artifacts[''data''].path}}'
+        - --model
+        - '{{$.inputs.artifacts[''model''].path}}'
+        - --label-column-name
+        - '{{$.inputs.parameters[''label_column_name'']}}'
+        - --predictions
+        - '{{$.outputs.artifacts[''predictions''].path}}'
+    exec-xgboost-predict-3:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+          'xgboost==1.1.1' 'pandas==1.0.5' 'pyarrow==0.17.1' || PIP_DISABLE_PIP_VERSION_CHECK=1
+          python3 -m pip install --quiet --no-warn-script-location 'xgboost==1.1.1'
+          'pandas==1.0.5' 'pyarrow==0.17.1' --user) && "$0" "$@"
+        - python3
+        - -u
+        - -c
+        - "def _make_parent_dirs_and_return_path(file_path: str):\n    import os\n\
+          \    os.makedirs(os.path.dirname(file_path), exist_ok=True)\n    return\
+          \ file_path\n\ndef xgboost_predict(\n    data_path,\n    model_path,\n \
+          \   predictions_path,\n    label_column_name = None,\n):\n    '''Make predictions\
+          \ using a trained XGBoost model.\n\n    Args:\n        data_path: Path for\
+          \ the feature data in Apache Parquet format.\n        model_path: Path for\
+          \ the trained model in binary XGBoost format.\n        predictions_path:\
+          \ Output path for the predictions.\n        label_column_name: Optional.\
+          \ Name of the column containing the label data that is excluded during the\
+          \ prediction.\n\n    Annotations:\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>\n\
+          \    '''\n    from pathlib import Path\n\n    import numpy\n    import pandas\n\
+          \    import xgboost\n\n    # Loading data\n    df = pandas.read_parquet(data_path)\n\
+          \    if label_column_name:\n        df = df.drop(columns=[label_column_name])\n\
+          \n    evaluation_data = xgboost.DMatrix(\n        data=df,\n    )\n\n  \
+          \  # Training\n    model = xgboost.Booster(model_file=model_path)\n\n  \
+          \  predictions = model.predict(evaluation_data)\n\n    Path(predictions_path).parent.mkdir(parents=True,\
+          \ exist_ok=True)\n    numpy.savetxt(predictions_path, predictions)\n\nimport\
+          \ argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make\
+          \ predictions using a trained XGBoost model.\\n\\n    Args:\\n        data_path:\
+          \ Path for the feature data in Apache Parquet format.\\n        model_path:\
+          \ Path for the trained model in binary XGBoost format.\\n        predictions_path:\
+          \ Output path for the predictions.\\n        label_column_name: Optional.\
+          \ Name of the column containing the label data that is excluded during the\
+          \ prediction.\\n\\n    Annotations:\\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>')\n\
+          _parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"\
+          model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"\
+          --label-column-name\", dest=\"label_column_name\", type=str, required=False,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"\
+          predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\
+          \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\
+          \n_outputs = xgboost_predict(**_parsed_args)\n"
+        args:
+        - --data
+        - '{{$.inputs.artifacts[''data''].path}}'
+        - --model
+        - '{{$.inputs.artifacts[''model''].path}}'
+        - --label-column-name
+        - '{{$.inputs.parameters[''label_column_name'']}}'
+        - --predictions
+        - '{{$.outputs.artifacts[''predictions''].path}}'
+    exec-xgboost-predict-4:
+      container:
+        image: python:3.7
+        command:
+        - sh
+        - -c
+        - (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
+          'xgboost==1.1.1' 'pandas==1.0.5' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3
+          -m pip install --quiet --no-warn-script-location 'xgboost==1.1.1' 'pandas==1.0.5'
+          --user) && "$0" "$@"
+        - python3
+        - -u
+        - -c
+        - "def _make_parent_dirs_and_return_path(file_path: str):\n    import os\n\
+          \    os.makedirs(os.path.dirname(file_path), exist_ok=True)\n    return\
+          \ file_path\n\ndef xgboost_predict(\n    data_path,  # Also supports LibSVM\n\
+          \    model_path,\n    predictions_path,\n    label_column = None,\n):\n\
+          \    '''Make predictions using a trained XGBoost model.\n\n    Args:\n \
+          \       data_path: Path for the feature data in CSV format.\n        model_path:\
+          \ Path for the trained model in binary XGBoost format.\n        predictions_path:\
+          \ Output path for the predictions.\n        label_column: Column containing\
+          \ the label data.\n\n    Annotations:\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>\n\
+          \    '''\n    from pathlib import Path\n\n    import numpy\n    import pandas\n\
+          \    import xgboost\n\n    df = pandas.read_csv(\n        data_path,\n \
+          \   )\n\n    if label_column is not None:\n        df = df.drop(columns=[df.columns[label_column]])\n\
+          \n    testing_data = xgboost.DMatrix(\n        data=df,\n    )\n\n    model\
+          \ = xgboost.Booster(model_file=model_path)\n\n    predictions = model.predict(testing_data)\n\
+          \n    Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n\
+          \    numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser\
+          \ = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions\
+          \ using a trained XGBoost model.\\n\\n    Args:\\n        data_path: Path\
+          \ for the feature data in CSV format.\\n        model_path: Path for the\
+          \ trained model in binary XGBoost format.\\n        predictions_path: Output\
+          \ path for the predictions.\\n        label_column: Column containing the\
+          \ label data.\\n\\n    Annotations:\\n        author: Alexey Volkov <alexey.volkov@ark-kun.com>')\n\
+          _parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True,\
+          \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"\
+          model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"\
+          --label-column\", dest=\"label_column\", type=int, required=False, default=argparse.SUPPRESS)\n\
+          _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\
+          \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\
+          \n_outputs = xgboost_predict(**_parsed_args)\n"
+        args:
+        - --data
+        - '{{$.inputs.artifacts[''data''].path}}'
+        - --model
+        - '{{$.inputs.artifacts[''model''].path}}'
+        - --label-column
+        - '{{$.inputs.parameters[''label_column'']}}'
+        - --predictions
+        - '{{$.outputs.artifacts[''predictions''].path}}'
+components:
+  comp-chicago-taxi-trips-dataset:
+    inputDefinitions:
+      parameters:
+        where:
+          parameterType: STRING
+        limit:
+          parameterType: NUMBER_INTEGER
+        select:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        table:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-chicago-taxi-trips-dataset
+  comp-xgboost-train:
+    inputDefinitions:
+      artifacts:
+        training_data:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+      parameters:
+        label_column:
+          parameterType: NUMBER_INTEGER
+        num_iterations:
+          parameterType: NUMBER_INTEGER
+        objective:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        model:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        model_config:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-xgboost-train
+  comp-xgboost-predict:
+    inputDefinitions:
+      artifacts:
+        data:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        model:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+      parameters:
+        label_column:
+          parameterType: NUMBER_INTEGER
+    outputDefinitions:
+      artifacts:
+        predictions:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-xgboost-predict
+  comp-convert-csv-to-apache-parquet:
+    inputDefinitions:
+      artifacts:
+        data:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    outputDefinitions:
+      artifacts:
+        output_data:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-convert-csv-to-apache-parquet
+  comp-xgboost-train-2:
+    inputDefinitions:
+      artifacts:
+        training_data:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+      parameters:
+        label_column_name:
+          parameterType: STRING
+        num_iterations:
+          parameterType: NUMBER_INTEGER
+        objective:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        model:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        model_config:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-xgboost-train-2
+  comp-xgboost-predict-2:
+    inputDefinitions:
+      artifacts:
+        data:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        model:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+      parameters:
+        label_column_name:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        predictions:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-xgboost-predict-2
+  comp-xgboost-predict-3:
+    inputDefinitions:
+      artifacts:
+        data:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        model:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+      parameters:
+        label_column_name:
+          parameterType: STRING
+    outputDefinitions:
+      artifacts:
+        predictions:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-xgboost-predict-3
+  comp-xgboost-predict-4:
+    inputDefinitions:
+      artifacts:
+        data:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+        model:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+      parameters:
+        label_column:
+          parameterType: NUMBER_INTEGER
+    outputDefinitions:
+      artifacts:
+        predictions:
+          artifactType:
+            schemaTitle: system.Artifact
+            schemaVersion: 0.0.1
+    executorLabel: exec-xgboost-predict-4
+root:
+  dag:
+    tasks:
+      chicago-taxi-trips-dataset:
+        taskInfo:
+          name: chicago-taxi-trips-dataset
+        inputs:
+          parameters:
+            where:
+              runtimeValue:
+                constant: trip_start_timestamp >= "2019-01-01" AND trip_start_timestamp
+                  < "2019-02-01"
+            select:
+              runtimeValue:
+                constant: tips,trip_seconds,trip_miles,pickup_community_area,dropoff_community_area,fare,tolls,extras,trip_total
+            limit:
+              runtimeValue:
+                constant: 10000.0
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-chicago-taxi-trips-dataset
+      xgboost-train:
+        taskInfo:
+          name: xgboost-train
+        inputs:
+          parameters:
+            label_column:
+              runtimeValue:
+                constant: 0.0
+            objective:
+              runtimeValue:
+                constant: reg:squarederror
+            num_iterations:
+              runtimeValue:
+                constant: 200.0
+          artifacts:
+            training_data:
+              taskOutputArtifact:
+                producerTask: chicago-taxi-trips-dataset
+                outputArtifactKey: table
+        dependentTasks:
+        - chicago-taxi-trips-dataset
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-xgboost-train
+      xgboost-predict:
+        taskInfo:
+          name: xgboost-predict
+        inputs:
+          parameters:
+            label_column:
+              runtimeValue:
+                constant: 0.0
+          artifacts:
+            data:
+              taskOutputArtifact:
+                producerTask: chicago-taxi-trips-dataset
+                outputArtifactKey: table
+            model:
+              taskOutputArtifact:
+                producerTask: xgboost-train
+                outputArtifactKey: model
+        dependentTasks:
+        - chicago-taxi-trips-dataset
+        - xgboost-train
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-xgboost-predict
+      convert-csv-to-apache-parquet:
+        taskInfo:
+          name: convert-csv-to-apache-parquet
+        inputs:
+          artifacts:
+            data:
+              taskOutputArtifact:
+                producerTask: chicago-taxi-trips-dataset
+                outputArtifactKey: table
+        dependentTasks:
+        - chicago-taxi-trips-dataset
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-convert-csv-to-apache-parquet
+      xgboost-train-2:
+        taskInfo:
+          name: xgboost-train-2
+        inputs:
+          parameters:
+            label_column_name:
+              runtimeValue:
+                constant: tips
+            objective:
+              runtimeValue:
+                constant: reg:squarederror
+            num_iterations:
+              runtimeValue:
+                constant: 200.0
+          artifacts:
+            training_data:
+              taskOutputArtifact:
+                producerTask: convert-csv-to-apache-parquet
+                outputArtifactKey: output_data
+        dependentTasks:
+        - convert-csv-to-apache-parquet
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-xgboost-train-2
+      xgboost-predict-2:
+        taskInfo:
+          name: xgboost-predict-2
+        inputs:
+          parameters:
+            label_column_name:
+              runtimeValue:
+                constant: tips
+          artifacts:
+            data:
+              taskOutputArtifact:
+                producerTask: convert-csv-to-apache-parquet
+                outputArtifactKey: output_data
+            model:
+              taskOutputArtifact:
+                producerTask: xgboost-train-2
+                outputArtifactKey: model
+        dependentTasks:
+        - convert-csv-to-apache-parquet
+        - xgboost-train-2
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-xgboost-predict-2
+      xgboost-predict-3:
+        taskInfo:
+          name: xgboost-predict-3
+        inputs:
+          parameters:
+            label_column_name:
+              runtimeValue:
+                constant: tips
+          artifacts:
+            data:
+              taskOutputArtifact:
+                producerTask: convert-csv-to-apache-parquet
+                outputArtifactKey: output_data
+            model:
+              taskOutputArtifact:
+                producerTask: xgboost-train
+                outputArtifactKey: model
+        dependentTasks:
+        - convert-csv-to-apache-parquet
+        - xgboost-train
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-xgboost-predict-3
+      xgboost-predict-4:
+        taskInfo:
+          name: xgboost-predict-4
+        inputs:
+          parameters:
+            label_column:
+              runtimeValue:
+                constant: 0.0
+          artifacts:
+            data:
+              taskOutputArtifact:
+                producerTask: chicago-taxi-trips-dataset
+                outputArtifactKey: table
+            model:
+              taskOutputArtifact:
+                producerTask: xgboost-train-2
+                outputArtifactKey: model
+        dependentTasks:
+        - chicago-taxi-trips-dataset
+        - xgboost-train-2
+        cachingOptions:
+          enableCache: true
+        componentRef:
+          name: comp-xgboost-predict-4
+defaultPipelineRoot: dummy_root