kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[SDK] Relax k8s sanitization (#2634) 

* update

* add allow_capital

* fix

* fix volume_ops sample

* fix pipeline name sanitization

* fix unittests

* fix sanitization in _client.py

* fix component output sanitization
This commit is contained in:
Jiaxiao Zheng 2019-11-26 10:28:10 -08:00 committed by IronPan
parent 533ed1d6f9
commit 790fe99aca
14 changed files with 107 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
samples/core/loop_static/loop_static.py
View File

@ -18,20 +18,20 @@ import kfp
@dsl.pipeline(name='my-pipeline')
def pipeline(my_pipe_param=10):
loop_args = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]
loop_args = [{'A_a': 1, 'B_b': 2}, {'A_a': 10, 'B_b': 20}]
with dsl.ParallelFor(loop_args) as item:
op1 = dsl.ContainerOp(
name="my-in-coop1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo op1 %s %s" % (item.a, my_pipe_param)],
arguments=["echo op1 %s %s" % (item.A_a, my_pipe_param)],
)
op2 = dsl.ContainerOp(
name="my-in-coop2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo op2 %s" % item.b],
arguments=["echo op2 %s" % item.B_b],
)
op_out = dsl.ContainerOp(

2
samples/core/volume_ops/volume_ops.py
View File

@ -22,7 +22,7 @@ import kfp.dsl as dsl
)
def volumeop_basic(size):
vop = dsl.VolumeOp(
name="create_pvc",
name="create-pvc",
resource_name="my-pvc",
modes=dsl.VOLUME_MODE_RWO,
size=size

5
sdk/python/kfp/_client.py
View File

@ -312,8 +312,9 @@ class Client(object):
if pipeline_package_path:
pipeline_obj = self._extract_pipeline_yaml(pipeline_package_path)
pipeline_json_string = json.dumps(pipeline_obj)
api_params = [kfp_server_api.ApiParameter(name=sanitize_k8s_name(k), value=str(v))
for k,v in params.items()]
api_params = [kfp_server_api.ApiParameter(
name=sanitize_k8s_name(name=k, allow_capital_underscore=True),
value=str(v)) for k,v in params.items()]
key = kfp_server_api.models.ApiResourceKey(id=experiment_id,
type=kfp_server_api.models.ApiResourceType.EXPERIMENT)
reference = kfp_server_api.models.ApiResourceReference(key, kfp_server_api.models.ApiRelationship.OWNER)

15
sdk/python/kfp/compiler/_k8s_helper.py
View File

@ -17,11 +17,22 @@ import re
from .. import dsl
def sanitize_k8s_name(name):
def sanitize_k8s_name(name, allow_capital_underscore=False):
"""From _make_kubernetes_name
sanitize_k8s_name cleans and converts the names in the workflow.
Args:
name: original name,
allow_capital_underscore: whether to allow capital letter and underscore
in this name.
Returns:
sanitized name.
"""
return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-')
if allow_capital_underscore:
return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-')
else:
return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-')
def convert_k8s_obj_to_json(k8s_obj):

14
sdk/python/kfp/compiler/compiler.py
View File

@ -494,7 +494,7 @@ class Compiler(object):
for argument_set in loop_tasks:
c_dict = {}
for k, v in argument_set.items():
c_dict[sanitize_k8s_name(k)] = v
c_dict[sanitize_k8s_name(k, True)] = v
sanitized_tasks.append(c_dict)
else:
sanitized_tasks = loop_tasks
@ -698,23 +698,23 @@ class Compiler(object):
sanitized_name = sanitize_k8s_name(op.name)
op.name = sanitized_name
for param in op.outputs.values():
param.name = sanitize_k8s_name(param.name)
param.name = sanitize_k8s_name(param.name, True)
if param.op_name:
param.op_name = sanitize_k8s_name(param.op_name)
if op.output is not None and not isinstance(op.output, dsl._container_op._MultipleOutputsError):
op.output.name = sanitize_k8s_name(op.output.name)
op.output.name = sanitize_k8s_name(op.output.name, True)
op.output.op_name = sanitize_k8s_name(op.output.op_name)
if op.dependent_names:
op.dependent_names = [sanitize_k8s_name(name) for name in op.dependent_names]
if isinstance(op, dsl.ContainerOp) and op.file_outputs is not None:
sanitized_file_outputs = {}
for key in op.file_outputs.keys():
sanitized_file_outputs[sanitize_k8s_name(key)] = op.file_outputs[key]
sanitized_file_outputs[sanitize_k8s_name(key, True)] = op.file_outputs[key]
op.file_outputs = sanitized_file_outputs
elif isinstance(op, dsl.ResourceOp) and op.attribute_outputs is not None:
sanitized_attribute_outputs = {}
for key in op.attribute_outputs.keys():
sanitized_attribute_outputs[sanitize_k8s_name(key)] = \
sanitized_attribute_outputs[sanitize_k8s_name(key, True)] = \
op.attribute_outputs[key]
op.attribute_outputs = sanitized_attribute_outputs
sanitized_ops[sanitized_name] = op
@ -757,7 +757,7 @@ class Compiler(object):
if arg_name == input.name:
arg_type = input.type
break
args_list.append(dsl.PipelineParam(sanitize_k8s_name(arg_name), param_type=arg_type))
args_list.append(dsl.PipelineParam(sanitize_k8s_name(arg_name, True), param_type=arg_type))
with dsl.Pipeline(pipeline_name) as dsl_pipeline:
pipeline_func(*args_list)
@ -770,7 +770,7 @@ class Compiler(object):
# Fill in the default values.
args_list_with_defaults = []
if pipeline_meta.inputs:
args_list_with_defaults = [dsl.PipelineParam(sanitize_k8s_name(arg_name))
args_list_with_defaults = [dsl.PipelineParam(sanitize_k8s_name(arg_name, True))
for arg_name in argspec.args]
if argspec.defaults:
for arg, default in zip(reversed(args_list_with_defaults), reversed(argspec.defaults)):

21
sdk/python/kfp/dsl/_pipeline_param.py
View File

@ -22,10 +22,21 @@ ConditionOperator = namedtuple('ConditionOperator', 'operator operand1 operand2'
PipelineParamTuple = namedtuple('PipelineParamTuple', 'name op pattern')
def sanitize_k8s_name(name):
"""From _make_kubernetes_name
sanitize_k8s_name cleans and converts the names in the workflow.
"""
def sanitize_k8s_name(name, allow_capital_underscore=False):
"""From _make_kubernetes_name
sanitize_k8s_name cleans and converts the names in the workflow.
Args:
name: original name,
allow_capital_underscore: whether to allow capital letter and underscore
in this name.
Returns:
sanitized name.
"""
if allow_capital_underscore:
return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', name)).lstrip('-').rstrip('-')
else:
return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', name.lower())).lstrip('-').rstrip('-')
@ -42,7 +53,7 @@ def match_serialized_pipelineparam(payload: str):
for match in matches:
pattern = '{{pipelineparam:op=%s;name=%s}}' % (match[0], match[1])
param_tuples.append(PipelineParamTuple(
name=sanitize_k8s_name(match[1]),
name=sanitize_k8s_name(match[1], True),
op=sanitize_k8s_name(match[0]),
pattern=pattern))
return param_tuples

12
sdk/python/tests/compiler/testdata/pipelineparams.yaml vendored
View File

@ -25,18 +25,18 @@ spec:
parameters:
- name: tag
value: latest
- name: sleep-ms
- name: sleep_ms
value: '10'
templates:
- name: download
inputs:
parameters:
- name: sleep-ms
- name: sleep_ms
- name: tag
container:
image: busybox:{{inputs.parameters.tag}}
args:
- sleep {{inputs.parameters.sleep-ms}}; wget localhost:5678 -O /tmp/results.txt
- sleep {{inputs.parameters.sleep_ms}}; wget localhost:5678 -O /tmp/results.txt
command:
- sh
- "-c"
@ -70,15 +70,15 @@ spec:
- name: pipelineparams
inputs:
parameters:
- name: sleep-ms
- name: sleep_ms
- name: tag
dag:
tasks:
- name: download
arguments:
parameters:
- name: sleep-ms
value: "{{inputs.parameters.sleep-ms}}"
- name: sleep_ms
value: "{{inputs.parameters.sleep_ms}}"
- name: tag
value: "{{inputs.parameters.tag}}"
template: download

2
sdk/python/tests/compiler/testdata/recursive_while.yaml vendored
View File

@ -8,7 +8,7 @@ metadata:
spec:
arguments:
parameters:
- name: maxval
- name: maxVal
value: '12'
entrypoint: pipeline-flip-coin
serviceAccountName: pipeline-runner

12
sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml vendored
View File

@ -8,7 +8,7 @@ metadata:
spec:
arguments:
parameters:
- name: rok-url
- name: rok_url
entrypoint: volumesnapshotop-rokurl
serviceAccountName: pipeline-runner
templates:
@ -56,7 +56,7 @@ spec:
successCondition: status.readyToUse == true
- inputs:
parameters:
- name: rok-url
- name: rok_url
name: create-volume-1
outputs:
parameters:
@ -72,7 +72,7 @@ spec:
resource:
action: create
manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n annotations:\n\
\ rok/origin: '{{inputs.parameters.rok-url}}'\n name: '{{workflow.name}}-vol1'\n\
\ rok/origin: '{{inputs.parameters.rok_url}}'\n name: '{{workflow.name}}-vol1'\n\
spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \
\ storage: 1Gi\n"
- inputs:
@ -192,8 +192,8 @@ spec:
template: create-snapshot-2
- arguments:
parameters:
- name: rok-url
value: '{{inputs.parameters.rok-url}}'
- name: rok_url
value: '{{inputs.parameters.rok_url}}'
name: create-volume-1
template: create-volume-1
- arguments:
@ -242,5 +242,5 @@ spec:
template: step3-output
inputs:
parameters:
- name: rok-url
- name: rok_url
name: volumesnapshotop-rokurl

26
sdk/python/tests/compiler/testdata/withitem_basic.yaml vendored
View File

@ -8,7 +8,7 @@ metadata:
spec:
arguments:
parameters:
- name: my-pipe-param
- name: my_pipe_param
value: '10'
entrypoint: my-pipeline
serviceAccountName: pipeline-runner
@ -19,8 +19,8 @@ spec:
parameters:
- name: loop-item-param-00000001-subvar-a
value: '{{inputs.parameters.loop-item-param-00000001-subvar-a}}'
- name: my-pipe-param
value: '{{inputs.parameters.my-pipe-param}}'
- name: my_pipe_param
value: '{{inputs.parameters.my_pipe_param}}'
name: my-in-coop1
template: my-in-coop1
- arguments:
@ -33,11 +33,11 @@ spec:
parameters:
- name: loop-item-param-00000001-subvar-a
- name: loop-item-param-00000001-subvar-b
- name: my-pipe-param
- name: my_pipe_param
name: for-loop-for-loop-00000001-1
- container:
args:
- echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.my-pipe-param}}
- echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.my_pipe_param}}
command:
- sh
- -c
@ -45,7 +45,7 @@ spec:
inputs:
parameters:
- name: loop-item-param-00000001-subvar-a
- name: my-pipe-param
- name: my_pipe_param
name: my-in-coop1
- container:
args:
@ -60,14 +60,14 @@ spec:
name: my-in-coop2
- container:
args:
- echo {{inputs.parameters.my-pipe-param}}
- echo {{inputs.parameters.my_pipe_param}}
command:
- sh
- -c
image: library/bash:4.4.23
inputs:
parameters:
- name: my-pipe-param
- name: my_pipe_param
name: my-out-cop
- dag:
tasks:
@ -77,8 +77,8 @@ spec:
value: '{{item.a}}'
- name: loop-item-param-00000001-subvar-b
value: '{{item.b}}'
- name: my-pipe-param
value: '{{inputs.parameters.my-pipe-param}}'
- name: my_pipe_param
value: '{{inputs.parameters.my_pipe_param}}'
name: for-loop-for-loop-00000001-1
template: for-loop-for-loop-00000001-1
withItems:
@ -88,11 +88,11 @@ spec:
b: 20
- arguments:
parameters:
- name: my-pipe-param
value: '{{inputs.parameters.my-pipe-param}}'
- name: my_pipe_param
value: '{{inputs.parameters.my_pipe_param}}'
name: my-out-cop
template: my-out-cop
inputs:
parameters:
- name: my-pipe-param
- name: my_pipe_param
name: my-pipeline

40
sdk/python/tests/compiler/testdata/withitem_nested.yaml vendored
View File

@ -8,7 +8,7 @@ metadata:
spec:
arguments:
parameters:
- name: my-pipe-param
- name: my_pipe_param
value: '10'
entrypoint: my-pipeline
serviceAccountName: pipeline-runner
@ -21,8 +21,8 @@ spec:
value: '{{inputs.parameters.loop-item-param-00000001-subvar-a}}'
- name: loop-item-param-00000002
value: '{{item}}'
- name: my-pipe-param
value: '{{inputs.parameters.my-pipe-param}}'
- name: my_pipe_param
value: '{{inputs.parameters.my_pipe_param}}'
name: for-loop-for-loop-00000002-2
template: for-loop-for-loop-00000002-2
withItems:
@ -33,8 +33,8 @@ spec:
parameters:
- name: loop-item-param-00000001-subvar-a
value: '{{inputs.parameters.loop-item-param-00000001-subvar-a}}'
- name: my-pipe-param
value: '{{inputs.parameters.my-pipe-param}}'
- name: my_pipe_param
value: '{{inputs.parameters.my_pipe_param}}'
name: my-in-coop1
template: my-in-coop1
- arguments:
@ -47,7 +47,7 @@ spec:
parameters:
- name: loop-item-param-00000001-subvar-a
- name: loop-item-param-00000001-subvar-b
- name: my-pipe-param
- name: my_pipe_param
name: for-loop-for-loop-00000001-1
- dag:
tasks:
@ -57,19 +57,19 @@ spec:
value: '{{inputs.parameters.loop-item-param-00000001-subvar-a}}'
- name: loop-item-param-00000002
value: '{{inputs.parameters.loop-item-param-00000002}}'
- name: my-pipe-param
value: '{{inputs.parameters.my-pipe-param}}'
- name: my_pipe_param
value: '{{inputs.parameters.my_pipe_param}}'
name: my-inner-inner-coop
template: my-inner-inner-coop
inputs:
parameters:
- name: loop-item-param-00000001-subvar-a
- name: loop-item-param-00000002
- name: my-pipe-param
- name: my_pipe_param
name: for-loop-for-loop-00000002-2
- container:
args:
- echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.my-pipe-param}}
- echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.my_pipe_param}}
command:
- sh
- -c
@ -77,7 +77,7 @@ spec:
inputs:
parameters:
- name: loop-item-param-00000001-subvar-a
- name: my-pipe-param
- name: my_pipe_param
name: my-in-coop1
- container:
args:
@ -93,7 +93,7 @@ spec:
- container:
args:
- echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.loop-item-param-00000002}}
{{inputs.parameters.my-pipe-param}}
{{inputs.parameters.my_pipe_param}}
command:
- sh
- -c
@ -102,18 +102,18 @@ spec:
parameters:
- name: loop-item-param-00000001-subvar-a
- name: loop-item-param-00000002
- name: my-pipe-param
- name: my_pipe_param
name: my-inner-inner-coop
- container:
args:
- echo {{inputs.parameters.my-pipe-param}}
- echo {{inputs.parameters.my_pipe_param}}
command:
- sh
- -c
image: library/bash:4.4.23
inputs:
parameters:
- name: my-pipe-param
- name: my_pipe_param
name: my-out-cop
- dag:
tasks:
@ -123,8 +123,8 @@ spec:
value: '{{item.a}}'
- name: loop-item-param-00000001-subvar-b
value: '{{item.b}}'
- name: my-pipe-param
value: '{{inputs.parameters.my-pipe-param}}'
- name: my_pipe_param
value: '{{inputs.parameters.my_pipe_param}}'
name: for-loop-for-loop-00000001-1
template: for-loop-for-loop-00000001-1
withItems:
@ -134,11 +134,11 @@ spec:
b: 20
- arguments:
parameters:
- name: my-pipe-param
value: '{{inputs.parameters.my-pipe-param}}'
- name: my_pipe_param
value: '{{inputs.parameters.my_pipe_param}}'
name: my-out-cop
template: my-out-cop
inputs:
parameters:
- name: my-pipe-param
- name: my_pipe_param
name: my-pipeline

16
sdk/python/tests/compiler/testdata/withparam_global.yaml vendored
View File

@ -8,7 +8,7 @@ metadata:
spec:
arguments:
parameters:
- name: loopidy-doop
- name: loopidy_doop
value: '[3, 5, 7, 9]'
entrypoint: my-pipeline
serviceAccountName: pipeline-runner
@ -17,24 +17,24 @@ spec:
tasks:
- arguments:
parameters:
- name: loopidy-doop
value: '{{inputs.parameters.loopidy-doop}}'
- name: loopidy_doop
value: '{{inputs.parameters.loopidy_doop}}'
name: my-in-cop1
template: my-in-cop1
inputs:
parameters:
- name: loopidy-doop
- name: loopidy_doop
name: for-loop-for-loop-00000001-1
- container:
args:
- 'echo no output global op1, item: {{inputs.parameters.loopidy-doop}}'
- 'echo no output global op1, item: {{inputs.parameters.loopidy_doop}}'
command:
- sh
- -c
image: library/bash:4.4.23
inputs:
parameters:
- name: loopidy-doop
- name: loopidy_doop
name: my-in-cop1
- container:
args:
@ -68,13 +68,13 @@ spec:
tasks:
- arguments:
parameters:
- name: loopidy-doop
- name: loopidy_doop
value: '{{item}}'
dependencies:
- my-out-cop0
name: for-loop-for-loop-00000001-1
template: for-loop-for-loop-00000001-1
withParam: '{{workflow.parameters.loopidy-doop}}'
withParam: '{{workflow.parameters.loopidy_doop}}'
- name: my-out-cop0
template: my-out-cop0
- arguments:

16
sdk/python/tests/compiler/testdata/withparam_global_dict.yaml vendored
View File

@ -8,7 +8,7 @@ metadata:
spec:
arguments:
parameters:
- name: loopidy-doop
- name: loopidy_doop
value: '[{"a": 1, "b": 2}, {"a": 10, "b": 20}]'
entrypoint: my-pipeline
serviceAccountName: pipeline-runner
@ -17,24 +17,24 @@ spec:
tasks:
- arguments:
parameters:
- name: loopidy-doop-subvar-a
value: '{{inputs.parameters.loopidy-doop-subvar-a}}'
- name: loopidy_doop-subvar-a
value: '{{inputs.parameters.loopidy_doop-subvar-a}}'
name: my-in-cop1
template: my-in-cop1
inputs:
parameters:
- name: loopidy-doop-subvar-a
- name: loopidy_doop-subvar-a
name: for-loop-for-loop-00000001-1
- container:
args:
- 'echo no output global op1, item.a: {{inputs.parameters.loopidy-doop-subvar-a}}'
- 'echo no output global op1, item.a: {{inputs.parameters.loopidy_doop-subvar-a}}'
command:
- sh
- -c
image: library/bash:4.4.23
inputs:
parameters:
- name: loopidy-doop-subvar-a
- name: loopidy_doop-subvar-a
name: my-in-cop1
- container:
args:
@ -68,13 +68,13 @@ spec:
tasks:
- arguments:
parameters:
- name: loopidy-doop-subvar-a
- name: loopidy_doop-subvar-a
value: '{{item.a}}'
dependencies:
- my-out-cop0
name: for-loop-for-loop-00000001-1
template: for-loop-for-loop-00000001-1
withParam: '{{workflow.parameters.loopidy-doop}}'
withParam: '{{workflow.parameters.loopidy_doop}}'
- name: my-out-cop0
template: my-out-cop0
- arguments:

4
test/sample-test/configs/xgboost_training_cm.config.yaml
View File

@ -16,8 +16,8 @@ test_name: xgboost_training_cm
arguments:
output:
project: ml-pipeline-test
train-data: gs://ml-pipeline-dataset/sample-test/sfpd/train_20.csv
eval-data: gs://ml-pipeline-dataset/sample-test/sfpd/eval_5.csv
train_data: gs://ml-pipeline-dataset/sample-test/sfpd/train_20.csv
eval_data: gs://ml-pipeline-dataset/sample-test/sfpd/eval_5.csv
schema: gs://ml-pipeline-dataset/sample-test/sfpd/schema.json
rounds: 5
workers: 2