diff --git a/sdk/python/kfp/compiler/pipeline_spec_builder.py b/sdk/python/kfp/compiler/pipeline_spec_builder.py index 1c5fc41fdf..3b8b46bc77 100644 --- a/sdk/python/kfp/compiler/pipeline_spec_builder.py +++ b/sdk/python/kfp/compiler/pipeline_spec_builder.py @@ -410,8 +410,8 @@ def build_container_spec_for_task( container_spec = ( pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec( image=task.container_spec.image, - command=task.container_spec.commands, - args=task.container_spec.arguments, + command=task.container_spec.command, + args=task.container_spec.args, env=[ pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec .EnvVar(name=name, value=value) diff --git a/sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_env.py b/sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_env.py index 87383a03d8..2a3effc706 100644 --- a/sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_env.py +++ b/sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_env.py @@ -29,7 +29,7 @@ name: Print env implementation: container: image: alpine - commands: + command: - sh - -c - | diff --git a/sdk/python/kfp/components/base_component_test.py b/sdk/python/kfp/components/base_component_test.py index cd0a1f5f4f..3404278f09 100644 --- a/sdk/python/kfp/components/base_component_test.py +++ b/sdk/python/kfp/components/base_component_test.py @@ -32,7 +32,7 @@ component_op = TestComponent( name='component_1', implementation=structures.ContainerSpec( image='alpine', - commands=[ + command=[ 'sh', '-c', 'set -ex\necho "$0" "$1" "$2" > "$3"', diff --git a/sdk/python/kfp/components/component_factory.py b/sdk/python/kfp/components/component_factory.py index 8ea7e3055d..0a533a95cd 100644 --- a/sdk/python/kfp/components/component_factory.py +++ b/sdk/python/kfp/components/component_factory.py @@ -402,8 +402,8 @@ def create_component_from_func(func: Callable, component_spec.implementation = structures.Implementation( container=structures.ContainerSpec( image=component_image, - commands=packages_to_install_command + command, - arguments=args, + command=packages_to_install_command + command, + args=args, )) module_path = pathlib.Path(inspect.getsourcefile(func)) diff --git a/sdk/python/kfp/components/pipeline_task.py b/sdk/python/kfp/components/pipeline_task.py index 2124bab975..e85bb4cb6e 100644 --- a/sdk/python/kfp/components/pipeline_task.py +++ b/sdk/python/kfp/components/pipeline_task.py @@ -336,10 +336,10 @@ class PipelineTask: container_spec = component_spec.implementation.container resolved_container_spec = copy.deepcopy(container_spec) - resolved_container_spec.commands = expand_argument_list( - container_spec.commands) - resolved_container_spec.arguments = expand_argument_list( - container_spec.arguments) + resolved_container_spec.command = expand_argument_list( + container_spec.command) + resolved_container_spec.args = expand_argument_list( + container_spec.args) return resolved_container_spec diff --git a/sdk/python/kfp/components/pipeline_task_test.py b/sdk/python/kfp/components/pipeline_task_test.py index 9fbc578fb7..a5889ddaef 100644 --- a/sdk/python/kfp/components/pipeline_task_test.py +++ b/sdk/python/kfp/components/pipeline_task_test.py @@ -31,11 +31,11 @@ V2_YAML = textwrap.dedent("""\ implementation: container: image: alpine - commands: + command: - sh - -c - echo "$0" >> "$1" - arguments: + args: - {inputValue: input1} - {outputPath: output1} """) @@ -47,11 +47,11 @@ V2_YAML_IF_PLACEHOLDER = textwrap.dedent("""\ implementation: container: image: alpine - commands: + command: - sh - -c - echo "$0" "$1" - arguments: + args: - ifPresent: inputName: optional_input_1 then: @@ -70,7 +70,7 @@ V2_YAML_CONCAT_PLACEHOLDER = textwrap.dedent("""\ implementation: container: image: alpine - commands: + command: - sh - -c - echo "$0" @@ -86,8 +86,8 @@ class PipelineTaskTest(parameterized.TestCase): implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', - commands=['sh', '-c', 'echo "$0" >> "$1"'], - arguments=[ + command=['sh', '-c', 'echo "$0" >> "$1"'], + args=[ structures.InputValuePlaceholder(input_name='input1'), structures.OutputPathPlaceholder(output_name='output1'), ], @@ -107,8 +107,8 @@ class PipelineTaskTest(parameterized.TestCase): ) expected_container_spec = structures.ContainerSpec( image='alpine', - commands=['sh', '-c', 'echo "$0" >> "$1"'], - arguments=[ + command=['sh', '-c', 'echo "$0" >> "$1"'], + args=[ "{{$.inputs.parameters['input1']}}", "{{$.outputs.artifacts['output1'].path}}", ], @@ -155,8 +155,8 @@ class PipelineTaskTest(parameterized.TestCase): 'expected_container_spec': structures.ContainerSpec( image='alpine', - commands=['sh', '-c', 'echo "$0" "$1"'], - arguments=[ + command=['sh', '-c', 'echo "$0" "$1"'], + args=[ 'input: ', "{{$.inputs.parameters['optional_input_1']}}", ], @@ -169,8 +169,8 @@ class PipelineTaskTest(parameterized.TestCase): 'expected_container_spec': structures.ContainerSpec( image='alpine', - commands=['sh', '-c', 'echo "$0" "$1"'], - arguments=[ + command=['sh', '-c', 'echo "$0" "$1"'], + args=[ 'default: ', 'Hello world!', ], @@ -193,7 +193,7 @@ class PipelineTaskTest(parameterized.TestCase): def test_resolve_concat_placeholder(self): expected_container_spec = structures.ContainerSpec( image='alpine', - commands=[ + command=[ 'sh', '-c', 'echo "$0"', diff --git a/sdk/python/kfp/components/structures.py b/sdk/python/kfp/components/structures.py index 80aeda8eef..829f8a29e8 100644 --- a/sdk/python/kfp/components/structures.py +++ b/sdk/python/kfp/components/structures.py @@ -18,11 +18,11 @@ import itertools import json from typing import Any, Dict, Mapping, Optional, Sequence, Union -from kfp.deprecated.components import _components -from kfp.deprecated.components import structures as v1_structures -from kfp.components import utils import pydantic import yaml +from kfp.components import utils +from kfp.deprecated.components import _components +from kfp.deprecated.components import structures as v1_structures class BaseModel(pydantic.BaseModel): @@ -195,18 +195,18 @@ class ContainerSpec(BaseModel): Attributes: image: The container image. - commands: Optional; the container entrypoint. - arguments: Optional; the arguments to the container entrypoint. + command: Optional; the container entrypoint. + args: Optional; the arguments to the container entrypoint. env: Optional; the environment variables to be passed to the container. resources: Optional; the specification on the resource requirements. """ image: str - commands: Optional[Sequence[ValidCommandArgs]] = None - arguments: Optional[Sequence[ValidCommandArgs]] = None + command: Optional[Sequence[ValidCommandArgs]] = None + args: Optional[Sequence[ValidCommandArgs]] = None env: Optional[Mapping[str, ValidCommandArgs]] = None resources: Optional[ResourceSpec] = None - @pydantic.validator('commands', 'arguments', allow_reuse=True) + @pydantic.validator('command', 'args', allow_reuse=True) def empty_sequence(cls, v): if v == []: return None @@ -335,8 +335,8 @@ class ComponentSpec(BaseModel): except AttributeError: valid_outputs = [] - for arg in itertools.chain((containerSpec.commands or []), - (containerSpec.arguments or [])): + for arg in itertools.chain((containerSpec.command or []), + (containerSpec.args or [])): cls._check_valid_placeholder_reference(valid_inputs, valid_outputs, arg) @@ -451,11 +451,11 @@ class ComponentSpec(BaseModel): ) implementation = component_dict['implementation']['container'] - implementation['commands'] = [ + implementation['command'] = [ _transform_arg(command) for command in implementation.pop('command', []) ] - implementation['arguments'] = [ + implementation['args'] = [ _transform_arg(command) for command in implementation.pop('args', []) ] @@ -473,13 +473,13 @@ class ComponentSpec(BaseModel): return obj # Must assign these after the constructor call, otherwise it won't work. - if implementation['commands']: - container_spec.commands = [ - _copy_model(cmd) for cmd in implementation['commands'] + if implementation['command']: + container_spec.command = [ + _copy_model(cmd) for cmd in implementation['command'] ] - if implementation['arguments']: - container_spec.arguments = [ - _copy_model(arg) for arg in implementation['arguments'] + if implementation['args']: + container_spec.args = [ + _copy_model(arg) for arg in implementation['args'] ] if implementation['env']: container_spec.env = { @@ -551,11 +551,11 @@ class ComponentSpec(BaseModel): image=self.implementation.container.image, command=[ _transform_arg(cmd) - for cmd in self.implementation.container.commands or [] + for cmd in self.implementation.container.command or [] ], args=[ _transform_arg(arg) for arg in - self.implementation.container.arguments or [] + self.implementation.container.args or [] ], env={ name: _transform_arg(value) for name, value in @@ -574,10 +574,11 @@ class ComponentSpec(BaseModel): Returns: Component spec in the form of V2 ComponentSpec. """ + json_component = yaml.safe_load(component_yaml) try: return ComponentSpec.parse_obj(json_component) - except pydantic.ValidationError: + except (pydantic.ValidationError, AttributeError): v1_component = _components._load_component_spec_from_component_text( component_yaml) return cls.from_v1_component_spec(v1_component) diff --git a/sdk/python/kfp/components/structures_test.py b/sdk/python/kfp/components/structures_test.py index 6a6a1299d5..e846f01c9e 100644 --- a/sdk/python/kfp/components/structures_test.py +++ b/sdk/python/kfp/components/structures_test.py @@ -21,7 +21,6 @@ import pydantic from absl.testing import parameterized from kfp.components import structures - V1_YAML_IF_PLACEHOLDER = textwrap.dedent("""\ name: component_if inputs: @@ -48,7 +47,7 @@ V2_YAML_IF_PLACEHOLDER = textwrap.dedent("""\ implementation: container: image: alpine - arguments: + args: - ifPresent: inputName: optional_input_1 then: @@ -62,7 +61,7 @@ V2_COMPONENT_SPEC_IF_PLACEHOLDER = structures.ComponentSpec( implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', - arguments=[ + args=[ structures.IfPresentPlaceholder( if_structure=structures.IfPresentPlaceholderStructure( input_name='optional_input_1', @@ -99,7 +98,7 @@ V2_YAML_CONCAT_PLACEHOLDER = textwrap.dedent("""\ implementation: container: image: alpine - arguments: + args: - concat: - --arg1 - {inputValue: input_prefix} @@ -110,7 +109,7 @@ V2_COMPONENT_SPEC_CONCAT_PLACEHOLDER = structures.ComponentSpec( implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', - arguments=[ + args=[ structures.ConcatPlaceholder(concat=[ '--arg1', structures.InputValuePlaceholder(input_name='input_prefix'), @@ -126,7 +125,7 @@ V2_YAML_NESTED_PLACEHOLDER = textwrap.dedent("""\ implementation: container: image: alpine - arguments: + args: - concat: - --arg1 - ifPresent: @@ -147,7 +146,7 @@ V2_COMPONENT_SPEC_NESTED_PLACEHOLDER = structures.ComponentSpec( implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', - arguments=[ + args=[ structures.ConcatPlaceholder(concat=[ '--arg1', structures.IfPresentPlaceholder( @@ -185,7 +184,7 @@ class StructuresTest(parameterized.TestCase): implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', - commands=[ + command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', @@ -208,7 +207,7 @@ class StructuresTest(parameterized.TestCase): implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', - commands=[ + command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', @@ -233,7 +232,7 @@ class StructuresTest(parameterized.TestCase): implementation: container: image: alpine - commands: + command: - sh - -c - 'set -ex @@ -249,7 +248,7 @@ class StructuresTest(parameterized.TestCase): implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', - commands=[ + command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', @@ -306,7 +305,7 @@ class StructuresTest(parameterized.TestCase): implementation: container: image: alpine - commands: + command: - sh - -c - 'set -ex @@ -324,7 +323,7 @@ class StructuresTest(parameterized.TestCase): implementation=structures.Implementation( container=structures.ContainerSpec( image='alpine', - commands=[ + command=[ 'sh', '-c', 'set -ex\necho "$0" > "$1"', @@ -397,13 +396,13 @@ class StructuresTest(parameterized.TestCase): implementation=structures.Implementation( container=structures.ContainerSpec( image='busybox', - commands=[ + command=[ 'sh', '-c', (' mkdir -p $(dirname "$2") mkdir -p $(dirname "$3") ' 'echo "$0" > "$2" cp "$1" "$3" '), ], - arguments=[ + args=[ structures.InputValuePlaceholder( input_name='input_parameter'), structures.InputPathPlaceholder( diff --git a/sdk/python/kfp/components/test_data/simple_yaml.yaml b/sdk/python/kfp/components/test_data/simple_yaml.yaml index 7035bcd07a..8db8477f7b 100644 --- a/sdk/python/kfp/components/test_data/simple_yaml.yaml +++ b/sdk/python/kfp/components/test_data/simple_yaml.yaml @@ -6,7 +6,7 @@ outputs: implementation: container: image: alpine - commands: + command: - sh - -c - 'set -ex diff --git a/sdk/python/kfp/components/yaml_component_test.py b/sdk/python/kfp/components/yaml_component_test.py index 2df8ef0962..af41e51b33 100644 --- a/sdk/python/kfp/components/yaml_component_test.py +++ b/sdk/python/kfp/components/yaml_component_test.py @@ -32,7 +32,7 @@ SAMPLE_YAML = textwrap.dedent("""\ implementation: container: image: alpine - commands: + command: - sh - -c - 'set -ex