kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(sdk)!: rename commands, arguments parameters to command, args (#7391)

This commit is contained in:
Connor McCarthy 2022-03-08 22:39:11 -07:00 committed by GitHub
parent d695ec556a
commit 82d17ff616
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 62 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sdk/python/kfp/compiler/pipeline_spec_builder.py
View File

@ -410,8 +410,8 @@ def build_container_spec_for_task(
container_spec = (
pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec(
image=task.container_spec.image,
command=task.container_spec.commands,
args=task.container_spec.arguments,
command=task.container_spec.command,
args=task.container_spec.args,
env=[
pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec
.EnvVar(name=name, value=value)

2
sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_env.py
View File

@ -29,7 +29,7 @@ name: Print env
implementation:
container:
image: alpine
commands:
command:
- sh
- -c
- |

2
sdk/python/kfp/components/base_component_test.py
View File

@ -32,7 +32,7 @@ component_op = TestComponent(
name='component_1',
implementation=structures.ContainerSpec(
image='alpine',
commands=[
command=[
'sh',
'-c',
'set -ex\necho "$0" "$1" "$2" > "$3"',

4
sdk/python/kfp/components/component_factory.py
View File

@ -402,8 +402,8 @@ def create_component_from_func(func: Callable,
component_spec.implementation = structures.Implementation(
container=structures.ContainerSpec(
image=component_image,
commands=packages_to_install_command + command,
arguments=args,
command=packages_to_install_command + command,
args=args,
))
module_path = pathlib.Path(inspect.getsourcefile(func))

8
sdk/python/kfp/components/pipeline_task.py
View File

@ -336,10 +336,10 @@ class PipelineTask:
container_spec = component_spec.implementation.container
resolved_container_spec = copy.deepcopy(container_spec)
resolved_container_spec.commands = expand_argument_list(
container_spec.commands)
resolved_container_spec.arguments = expand_argument_list(
container_spec.arguments)
resolved_container_spec.command = expand_argument_list(
container_spec.command)
resolved_container_spec.args = expand_argument_list(
container_spec.args)
return resolved_container_spec

28
sdk/python/kfp/components/pipeline_task_test.py
View File

@ -31,11 +31,11 @@ V2_YAML = textwrap.dedent("""\
implementation:
container:
image: alpine
commands:
command:
- sh
- -c
- echo "$0" >> "$1"
arguments:
args:
- {inputValue: input1}
- {outputPath: output1}
""")
@ -47,11 +47,11 @@ V2_YAML_IF_PLACEHOLDER = textwrap.dedent("""\
implementation:
container:
image: alpine
commands:
command:
- sh
- -c
- echo "$0" "$1"
arguments:
args:
- ifPresent:
inputName: optional_input_1
then:
@ -70,7 +70,7 @@ V2_YAML_CONCAT_PLACEHOLDER = textwrap.dedent("""\
implementation:
container:
image: alpine
commands:
command:
- sh
- -c
- echo "$0"
@ -86,8 +86,8 @@ class PipelineTaskTest(parameterized.TestCase):
implementation=structures.Implementation(
container=structures.ContainerSpec(
image='alpine',
commands=['sh', '-c', 'echo "$0" >> "$1"'],
arguments=[
command=['sh', '-c', 'echo "$0" >> "$1"'],
args=[
structures.InputValuePlaceholder(input_name='input1'),
structures.OutputPathPlaceholder(output_name='output1'),
],
@ -107,8 +107,8 @@ class PipelineTaskTest(parameterized.TestCase):
)
expected_container_spec = structures.ContainerSpec(
image='alpine',
commands=['sh', '-c', 'echo "$0" >> "$1"'],
arguments=[
command=['sh', '-c', 'echo "$0" >> "$1"'],
args=[
"{{$.inputs.parameters['input1']}}",
"{{$.outputs.artifacts['output1'].path}}",
],
@ -155,8 +155,8 @@ class PipelineTaskTest(parameterized.TestCase):
'expected_container_spec':
structures.ContainerSpec(
image='alpine',
commands=['sh', '-c', 'echo "$0" "$1"'],
arguments=[
command=['sh', '-c', 'echo "$0" "$1"'],
args=[
'input: ',
"{{$.inputs.parameters['optional_input_1']}}",
],
@ -169,8 +169,8 @@ class PipelineTaskTest(parameterized.TestCase):
'expected_container_spec':
structures.ContainerSpec(
image='alpine',
commands=['sh', '-c', 'echo "$0" "$1"'],
arguments=[
command=['sh', '-c', 'echo "$0" "$1"'],
args=[
'default: ',
'Hello world!',
],
@ -193,7 +193,7 @@ class PipelineTaskTest(parameterized.TestCase):
def test_resolve_concat_placeholder(self):
expected_container_spec = structures.ContainerSpec(
image='alpine',
commands=[
command=[
'sh',
'-c',
'echo "$0"',

43
sdk/python/kfp/components/structures.py
View File

@ -18,11 +18,11 @@ import itertools
import json
from typing import Any, Dict, Mapping, Optional, Sequence, Union
from kfp.deprecated.components import _components
from kfp.deprecated.components import structures as v1_structures
from kfp.components import utils
import pydantic
import yaml
from kfp.components import utils
from kfp.deprecated.components import _components
from kfp.deprecated.components import structures as v1_structures
class BaseModel(pydantic.BaseModel):
@ -195,18 +195,18 @@ class ContainerSpec(BaseModel):
Attributes:
image: The container image.
commands: Optional; the container entrypoint.
arguments: Optional; the arguments to the container entrypoint.
command: Optional; the container entrypoint.
args: Optional; the arguments to the container entrypoint.
env: Optional; the environment variables to be passed to the container.
resources: Optional; the specification on the resource requirements.
"""
image: str
commands: Optional[Sequence[ValidCommandArgs]] = None
arguments: Optional[Sequence[ValidCommandArgs]] = None
command: Optional[Sequence[ValidCommandArgs]] = None
args: Optional[Sequence[ValidCommandArgs]] = None
env: Optional[Mapping[str, ValidCommandArgs]] = None
resources: Optional[ResourceSpec] = None
@pydantic.validator('commands', 'arguments', allow_reuse=True)
@pydantic.validator('command', 'args', allow_reuse=True)
def empty_sequence(cls, v):
if v == []:
return None
@ -335,8 +335,8 @@ class ComponentSpec(BaseModel):
except AttributeError:
valid_outputs = []
for arg in itertools.chain((containerSpec.commands or []),
(containerSpec.arguments or [])):
for arg in itertools.chain((containerSpec.command or []),
(containerSpec.args or [])):
cls._check_valid_placeholder_reference(valid_inputs, valid_outputs,
arg)
@ -451,11 +451,11 @@ class ComponentSpec(BaseModel):
)
implementation = component_dict['implementation']['container']
implementation['commands'] = [
implementation['command'] = [
_transform_arg(command)
for command in implementation.pop('command', [])
]
implementation['arguments'] = [
implementation['args'] = [
_transform_arg(command)
for command in implementation.pop('args', [])
]
@ -473,13 +473,13 @@ class ComponentSpec(BaseModel):
return obj
# Must assign these after the constructor call, otherwise it won't work.
if implementation['commands']:
container_spec.commands = [
_copy_model(cmd) for cmd in implementation['commands']
if implementation['command']:
container_spec.command = [
_copy_model(cmd) for cmd in implementation['command']
]
if implementation['arguments']:
container_spec.arguments = [
_copy_model(arg) for arg in implementation['arguments']
if implementation['args']:
container_spec.args = [
_copy_model(arg) for arg in implementation['args']
]
if implementation['env']:
container_spec.env = {
@ -551,11 +551,11 @@ class ComponentSpec(BaseModel):
image=self.implementation.container.image,
command=[
_transform_arg(cmd)
for cmd in self.implementation.container.commands or []
for cmd in self.implementation.container.command or []
],
args=[
_transform_arg(arg) for arg in
self.implementation.container.arguments or []
self.implementation.container.args or []
],
env={
name: _transform_arg(value) for name, value in
@ -574,10 +574,11 @@ class ComponentSpec(BaseModel):
Returns:
Component spec in the form of V2 ComponentSpec.
"""
json_component = yaml.safe_load(component_yaml)
try:
return ComponentSpec.parse_obj(json_component)
except pydantic.ValidationError:
except (pydantic.ValidationError, AttributeError):
v1_component = _components._load_component_spec_from_component_text(
component_yaml)
return cls.from_v1_component_spec(v1_component)

29
sdk/python/kfp/components/structures_test.py
View File

@ -21,7 +21,6 @@ import pydantic
from absl.testing import parameterized
from kfp.components import structures
V1_YAML_IF_PLACEHOLDER = textwrap.dedent("""\
name: component_if
inputs:
@ -48,7 +47,7 @@ V2_YAML_IF_PLACEHOLDER = textwrap.dedent("""\
implementation:
container:
image: alpine
arguments:
args:
- ifPresent:
inputName: optional_input_1
then:
@ -62,7 +61,7 @@ V2_COMPONENT_SPEC_IF_PLACEHOLDER = structures.ComponentSpec(
implementation=structures.Implementation(
container=structures.ContainerSpec(
image='alpine',
arguments=[
args=[
structures.IfPresentPlaceholder(
if_structure=structures.IfPresentPlaceholderStructure(
input_name='optional_input_1',
@ -99,7 +98,7 @@ V2_YAML_CONCAT_PLACEHOLDER = textwrap.dedent("""\
implementation:
container:
image: alpine
arguments:
args:
- concat:
- --arg1
- {inputValue: input_prefix}
@ -110,7 +109,7 @@ V2_COMPONENT_SPEC_CONCAT_PLACEHOLDER = structures.ComponentSpec(
implementation=structures.Implementation(
container=structures.ContainerSpec(
image='alpine',
arguments=[
args=[
structures.ConcatPlaceholder(concat=[
'--arg1',
structures.InputValuePlaceholder(input_name='input_prefix'),
@ -126,7 +125,7 @@ V2_YAML_NESTED_PLACEHOLDER = textwrap.dedent("""\
implementation:
container:
image: alpine
arguments:
args:
- concat:
- --arg1
- ifPresent:
@ -147,7 +146,7 @@ V2_COMPONENT_SPEC_NESTED_PLACEHOLDER = structures.ComponentSpec(
implementation=structures.Implementation(
container=structures.ContainerSpec(
image='alpine',
arguments=[
args=[
structures.ConcatPlaceholder(concat=[
'--arg1',
structures.IfPresentPlaceholder(
@ -185,7 +184,7 @@ class StructuresTest(parameterized.TestCase):
implementation=structures.Implementation(
container=structures.ContainerSpec(
image='alpine',
commands=[
command=[
'sh',
'-c',
'set -ex\necho "$0" > "$1"',
@ -208,7 +207,7 @@ class StructuresTest(parameterized.TestCase):
implementation=structures.Implementation(
container=structures.ContainerSpec(
image='alpine',
commands=[
command=[
'sh',
'-c',
'set -ex\necho "$0" > "$1"',
@ -233,7 +232,7 @@ class StructuresTest(parameterized.TestCase):
implementation:
container:
image: alpine
commands:
command:
- sh
- -c
- 'set -ex
@ -249,7 +248,7 @@ class StructuresTest(parameterized.TestCase):
implementation=structures.Implementation(
container=structures.ContainerSpec(
image='alpine',
commands=[
command=[
'sh',
'-c',
'set -ex\necho "$0" > "$1"',
@ -306,7 +305,7 @@ class StructuresTest(parameterized.TestCase):
implementation:
container:
image: alpine
commands:
command:
- sh
- -c
- 'set -ex
@ -324,7 +323,7 @@ class StructuresTest(parameterized.TestCase):
implementation=structures.Implementation(
container=structures.ContainerSpec(
image='alpine',
commands=[
command=[
'sh',
'-c',
'set -ex\necho "$0" > "$1"',
@ -397,13 +396,13 @@ class StructuresTest(parameterized.TestCase):
implementation=structures.Implementation(
container=structures.ContainerSpec(
image='busybox',
commands=[
command=[
'sh',
'-c',
(' mkdir -p $(dirname "$2") mkdir -p $(dirname "$3") '
'echo "$0" > "$2" cp "$1" "$3" '),
],
arguments=[
args=[
structures.InputValuePlaceholder(
input_name='input_parameter'),
structures.InputPathPlaceholder(

2
sdk/python/kfp/components/test_data/simple_yaml.yaml
View File

@ -6,7 +6,7 @@ outputs:
implementation:
container:
image: alpine
commands:
command:
- sh
- -c
- 'set -ex

2
sdk/python/kfp/components/yaml_component_test.py
View File

@ -32,7 +32,7 @@ SAMPLE_YAML = textwrap.dedent("""\
implementation:
container:
image: alpine
commands:
command:
- sh
- -c
- 'set -ex