kubeflow
/
kfp-tekton
mirror of https://github.com/kubeflow/kfp-tekton.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(sdk) add test for any-seq inside of loop (#927) 

* add test for any-seq inside of loop

* fix style: newline at the end of file

* recur when handling tekton pipeline variables

* update test name

* refactor adding a param

* add type to extracted task params inside loop

* fix style: no spaces in single-elem dict
This commit is contained in:
Michalina Kotwica 2022-05-04 18:10:32 +02:00 committed by GitHub
parent c854c27108
commit 284891c8b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 537 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
sdk/python/kfp_tekton/compiler/_tekton_handler.py
View File

@ -73,12 +73,18 @@ def _handle_tekton_pipeline_variables(pipeline_run):
'pipelineRun-uid': '$(context.pipelineRun.uid)'
}
add_type = pipeline_run.get('kind') != 'PipelineRun'
task_list = pipeline_run['spec']['pipelineSpec']['tasks']
for task in task_list:
if task.get('taskRef', {}):
continue
if 'taskSpec' in task and 'apiVersion' in task['taskSpec']:
# recur for embedded pipeline resources
if 'spec' in task['taskSpec'] and 'pipelineSpec' in task['taskSpec']['spec']:
resource_spec = task['taskSpec']
_handle_tekton_pipeline_variables(resource_spec)
continue
for key, val in pipeline_variables.items():
task_str = json.dumps(task['taskSpec']['steps'])
task_str = _process_argo_vars(task_str)
@ -90,11 +96,17 @@ def _handle_tekton_pipeline_variables(pipeline_run):
task['params'].append({'name': key, 'value': val})
else:
task['params'] = [{'name': key, 'value': val}]
if task['taskSpec'].get('params', ''):
if {'name': key} not in task['taskSpec']['params']:
task['taskSpec']['params'].append({'name': key})
else:
task['taskSpec']['params'] = [{'name': key}]
found = False
for param in task['taskSpec'].get('params', []):
if param['name'] == key:
found = True
break
if not found:
param = {'name': key}
if add_type:
param['type'] = 'string'
task['taskSpec'].setdefault('params', []).append(param)
return pipeline_run

7
sdk/python/tests/compiler/compiler_tests.py
View File

@ -713,6 +713,13 @@ class TestTektonCompiler(unittest.TestCase):
self._test_pipeline_workflow(any_sequence_pipeline, 'any_sequencer.yaml', skip_noninlined=True)
def test_any_sequencer_in_loop(self):
"""
Test any sequencer inside of a loop.
"""
from .testdata.any_sequencer_looped import any_sequencer_pipeline
self._test_pipeline_workflow(any_sequencer_pipeline, 'any_sequencer_looped.yaml')
def _test_pipeline_workflow_inlined_spec(self,
pipeline_function,
pipeline_yaml,

68
sdk/python/tests/compiler/testdata/any_sequencer_looped.py vendored Normal file
View File

@ -0,0 +1,68 @@
# Copyright 2021 kubeflow.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kfp import dsl
from kfp.components import load_component_from_text
from kfp_tekton.tekton import Loop, AnySequencer
from kfp_tekton.compiler import TektonCompiler
class Coder:
def empty(self):
return ""
TektonCompiler._get_unique_id_code = Coder.empty
def PrintOp(name: str, msg: str = None):
if msg is None:
msg = name
print_op = load_component_from_text(
"""
name: %s
inputs:
- {name: input_text, type: String, description: 'Represents an input parameter.'}
outputs:
- {name: output_value, type: String, description: 'Represents an output paramter.'}
implementation:
container:
image: alpine:3.6
command:
- sh
- -c
- |
set -e
echo $0 > $1
- {inputValue: input_text}
- {outputPath: output_value}
""" % (name)
)
return print_op(msg)
@dsl.pipeline("any-sequencer-looped")
def any_sequencer_pipeline(param: list = ["a", "b", "c"]):
op00 = PrintOp("print-00")
op01 = PrintOp("print-01")
AnySequencer([op00, op01], "any-seq-0", statusPath="status")
with Loop(param):
op10 = PrintOp("print-10")
op11 = PrintOp("print-11")
AnySequencer([op10, op11], "any-seq-1", statusPath="status")
if __name__ == '__main__':
TektonCompiler().compile(any_sequencer_pipeline, __file__.replace('.py', '.yaml'))

257
sdk/python/tests/compiler/testdata/any_sequencer_looped.yaml vendored Normal file
View File

@ -0,0 +1,257 @@
# Copyright 2021 kubeflow.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: tekton.dev/v1beta1
kind: PipelineRun
metadata:
name: any-sequencer-looped
annotations:
tekton.dev/output_artifacts: '{"any-seq-0": [{"key": "artifacts/$PIPELINERUN/any-seq-0/status.tgz",
"name": "any-seq-0-status", "path": "/tmp/outputs/status/data"}], "any-seq-1":
[{"key": "artifacts/$PIPELINERUN/any-seq-1/status.tgz", "name": "any-seq-1-status",
"path": "/tmp/outputs/status/data"}], "print-00": [{"key": "artifacts/$PIPELINERUN/print-00/output_value.tgz",
"name": "print-00-output_value", "path": "/tmp/outputs/output_value/data"}],
"print-01": [{"key": "artifacts/$PIPELINERUN/print-01/output_value.tgz", "name":
"print-01-output_value", "path": "/tmp/outputs/output_value/data"}], "print-10":
[{"key": "artifacts/$PIPELINERUN/print-10/output_value.tgz", "name": "print-10-output_value",
"path": "/tmp/outputs/output_value/data"}], "print-11": [{"key": "artifacts/$PIPELINERUN/print-11/output_value.tgz",
"name": "print-11-output_value", "path": "/tmp/outputs/output_value/data"}]}'
tekton.dev/input_artifacts: '{}'
tekton.dev/artifact_bucket: mlpipeline
tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
tekton.dev/artifact_endpoint_scheme: http://
tekton.dev/artifact_items: '{"any-seq-0": [["status", "$(results.status.path)"]],
"any-seq-1": [["status", "$(results.status.path)"]], "print-00": [["output_value",
"$(results.output-value.path)"]], "print-01": [["output_value", "$(results.output-value.path)"]],
"print-10": [["output_value", "$(results.output-value.path)"]], "print-11":
[["output_value", "$(results.output-value.path)"]]}'
sidecar.istio.io/inject: "false"
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[\"a\", \"b\",
\"c\"]", "name": "param", "optional": true, "type": "JsonArray"}], "name": "any-sequencer-looped"}'
spec:
params:
- name: param
value: '["a", "b", "c"]'
pipelineSpec:
params:
- name: param
default: '["a", "b", "c"]'
tasks:
- name: print-00
taskSpec:
steps:
- name: main
command:
- sh
- -c
- |
set -e
echo $0 > $1
- print-00
- $(results.output-value.path)
image: alpine:3.6
results:
- name: output-value
description: /tmp/outputs/output_value/data
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-00", "outputs":
[{"description": "Represents an output paramter.", "name": "output_value",
"type": "String"}], "version": "print-00@sha256=7c633686b0b637cfba6cba8c9b4e871cf172395432360d6aaa2d89623fb336b3"}'
tekton.dev/template: ''
timeout: 525600m
- name: print-01
taskSpec:
steps:
- name: main
command:
- sh
- -c
- |
set -e
echo $0 > $1
- print-01
- $(results.output-value.path)
image: alpine:3.6
results:
- name: output-value
description: /tmp/outputs/output_value/data
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-01", "outputs":
[{"description": "Represents an output paramter.", "name": "output_value",
"type": "String"}], "version": "print-01@sha256=d511ac628d43cc5b393fbebd10be93662b30117f1413b84afd4e7b2e5ff5ed33"}'
tekton.dev/template: ''
timeout: 525600m
- name: any-seq-0
taskSpec:
steps:
- name: main
args:
- --namespace
- $(params.pipelineRun-namespace)
- --prName
- $(params.pipelineRun-name)
- --taskList
- print-00,print-01
- --statusPath
- $(results.status.path)
command:
- any-task
image: dspipelines/any-sequencer:latest
results:
- name: status
description: /tmp/outputs/status/data
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-0", "outputs":
[{"description": "The output file to create the status", "name": "status"}],
"version": "any-seq-0@sha256=2a9bf729444af0ff906f5a6a77eff3eecfd419ebfbdb51e5f04cdfc11d58a7b8"}'
tekton.dev/template: ''
params:
- name: pipelineRun-name
- name: pipelineRun-namespace
timeout: 525600m
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: pipelineRun-namespace
value: $(context.pipelineRun.namespace)
- name: any-sequencer-looped-for-loop-1
params:
- name: param-loop-item
value: $(params.param)
taskSpec:
apiVersion: custom.tekton.dev/v1alpha1
kind: PipelineLoop
spec:
pipelineSpec:
params:
- name: param-loop-item
type: string
tasks:
- name: print-10
taskSpec:
steps:
- name: main
command:
- sh
- -c
- |
set -e
echo $0 > $1
- print-10
- $(results.output-value.path)
image: alpine:3.6
results:
- name: output-value
description: /tmp/outputs/output_value/data
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-10",
"outputs": [{"description": "Represents an output paramter.",
"name": "output_value", "type": "String"}], "version": "print-10@sha256=ea7fa8edfd92cf64a15f8464292143a4e0d5f34898368bfd63618884decf3be1"}'
tekton.dev/template: ''
timeout: 525600m
- name: print-11
taskSpec:
steps:
- name: main
command:
- sh
- -c
- |
set -e
echo $0 > $1
- print-11
- $(results.output-value.path)
image: alpine:3.6
results:
- name: output-value
description: /tmp/outputs/output_value/data
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-11",
"outputs": [{"description": "Represents an output paramter.",
"name": "output_value", "type": "String"}], "version": "print-11@sha256=3b423bb0ced0bc21fb9157ce814a38c94d3e72718a33cfa8b4ef4f75abe725bd"}'
tekton.dev/template: ''
timeout: 525600m
- name: any-seq-1
taskSpec:
steps:
- name: main
args:
- --namespace
- $(params.pipelineRun-namespace)
- --prName
- $(params.pipelineRun-name)
- --taskList
- print-10,print-11
- --statusPath
- $(results.status.path)
command:
- any-task
image: dspipelines/any-sequencer:latest
results:
- name: status
description: /tmp/outputs/status/data
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-1",
"outputs": [{"description": "The output file to create the status",
"name": "status"}], "version": "any-seq-1@sha256=75c65b3423dd61b79c35122746a1f16d67a98ab7b570daa36de2d310e1e57c22"}'
tekton.dev/template: ''
params:
- name: pipelineRun-name
type: string
- name: pipelineRun-namespace
type: string
timeout: 525600m
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: pipelineRun-namespace
value: $(context.pipelineRun.namespace)
iterateParam: param-loop-item
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
timeout: 525600m

188
sdk/python/tests/compiler/testdata/any_sequencer_looped_noninlined.yaml vendored Normal file
View File

@ -0,0 +1,188 @@
# Copyright 2021 kubeflow.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: tekton.dev/v1beta1
kind: PipelineRun
metadata:
name: any-sequencer-looped
annotations:
tekton.dev/output_artifacts: '{"any-seq-0": [{"key": "artifacts/$PIPELINERUN/any-seq-0/status.tgz",
"name": "any-seq-0-status", "path": "/tmp/outputs/status/data"}], "any-seq-1":
[{"key": "artifacts/$PIPELINERUN/any-seq-1/status.tgz", "name": "any-seq-1-status",
"path": "/tmp/outputs/status/data"}], "print-00": [{"key": "artifacts/$PIPELINERUN/print-00/output_value.tgz",
"name": "print-00-output_value", "path": "/tmp/outputs/output_value/data"}],
"print-01": [{"key": "artifacts/$PIPELINERUN/print-01/output_value.tgz", "name":
"print-01-output_value", "path": "/tmp/outputs/output_value/data"}], "print-10":
[{"key": "artifacts/$PIPELINERUN/print-10/output_value.tgz", "name": "print-10-output_value",
"path": "/tmp/outputs/output_value/data"}], "print-11": [{"key": "artifacts/$PIPELINERUN/print-11/output_value.tgz",
"name": "print-11-output_value", "path": "/tmp/outputs/output_value/data"}]}'
tekton.dev/input_artifacts: '{}'
tekton.dev/artifact_bucket: mlpipeline
tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
tekton.dev/artifact_endpoint_scheme: http://
tekton.dev/artifact_items: '{"any-seq-0": [["status", "$(results.status.path)"]],
"any-seq-1": [["status", "$(results.status.path)"]], "print-00": [["output_value",
"$(results.output-value.path)"]], "print-01": [["output_value", "$(results.output-value.path)"]],
"print-10": [["output_value", "$(results.output-value.path)"]], "print-11":
[["output_value", "$(results.output-value.path)"]]}'
sidecar.istio.io/inject: "false"
pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[\"a\", \"b\",
\"c\"]", "name": "param", "optional": true, "type": "JsonArray"}], "name": "any-sequencer-looped"}'
tekton.dev/resource_templates: '[{"apiVersion": "custom.tekton.dev/v1alpha1",
"kind": "PipelineLoop", "metadata": {"name": "any-sequencer-looped-for-loop-1"},
"spec": {"iterateParam": "param-loop-item", "pipelineSpec": {"params": [{"name":
"param-loop-item", "type": "string"}], "tasks": [{"name": "print-10", "taskSpec":
{"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
"{\"name\": \"print-10\", \"outputs\": [{\"description\": \"Represents an output
paramter.\", \"name\": \"output_value\", \"type\": \"String\"}], \"version\":
\"print-10@sha256=ea7fa8edfd92cf64a15f8464292143a4e0d5f34898368bfd63618884decf3be1\"}",
"tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
"true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
""}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
"output-value"}], "steps": [{"command": ["sh", "-c", "set -e\necho $0 > $1\n",
"print-10", "$(results.output-value.path)"], "image": "alpine:3.6", "name":
"main"}]}, "timeout": "525600m"}, {"name": "print-11", "taskSpec": {"metadata":
{"annotations": {"pipelines.kubeflow.org/component_spec_digest": "{\"name\":
\"print-11\", \"outputs\": [{\"description\": \"Represents an output paramter.\",
\"name\": \"output_value\", \"type\": \"String\"}], \"version\": \"print-11@sha256=3b423bb0ced0bc21fb9157ce814a38c94d3e72718a33cfa8b4ef4f75abe725bd\"}",
"tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
"true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
""}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
"output-value"}], "steps": [{"command": ["sh", "-c", "set -e\necho $0 > $1\n",
"print-11", "$(results.output-value.path)"], "image": "alpine:3.6", "name":
"main"}]}, "timeout": "525600m"}, {"name": "any-seq-1", "params": [{"name":
"pipelineRun-name", "value": "$(context.pipelineRun.name)"}, {"name": "pipelineRun-namespace",
"value": "$(context.pipelineRun.namespace)"}], "taskSpec": {"metadata": {"annotations":
{"pipelines.kubeflow.org/component_spec_digest": "{\"name\": \"any-seq-1\",
\"outputs\": [{\"description\": \"The output file to create the status\", \"name\":
\"status\"}], \"version\": \"any-seq-1@sha256=75c65b3423dd61b79c35122746a1f16d67a98ab7b570daa36de2d310e1e57c22\"}",
"tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
"true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
""}}, "params": [{"name": "pipelineRun-name", "type": "string"}, {"name": "pipelineRun-namespace",
"type": "string"}], "results": [{"description": "/tmp/outputs/status/data",
"name": "status"}], "steps": [{"args": ["--namespace", "$(params.pipelineRun-namespace)",
"--prName", "$(params.pipelineRun-name)", "--taskList", "print-10,print-11",
"--statusPath", "$(results.status.path)"], "command": ["any-task"], "image":
"dspipelines/any-sequencer:latest", "name": "main"}]}, "timeout": "525600m"}]}}}]'
spec:
params:
- name: param
value: '["a", "b", "c"]'
pipelineSpec:
params:
- name: param
default: '["a", "b", "c"]'
tasks:
- name: print-00
taskSpec:
steps:
- name: main
command:
- sh
- -c
- |
set -e
echo $0 > $1
- print-00
- $(results.output-value.path)
image: alpine:3.6
results:
- name: output-value
description: /tmp/outputs/output_value/data
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-00", "outputs":
[{"description": "Represents an output paramter.", "name": "output_value",
"type": "String"}], "version": "print-00@sha256=7c633686b0b637cfba6cba8c9b4e871cf172395432360d6aaa2d89623fb336b3"}'
tekton.dev/template: ''
timeout: 525600m
- name: print-01
taskSpec:
steps:
- name: main
command:
- sh
- -c
- |
set -e
echo $0 > $1
- print-01
- $(results.output-value.path)
image: alpine:3.6
results:
- name: output-value
description: /tmp/outputs/output_value/data
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec_digest: '{"name": "print-01", "outputs":
[{"description": "Represents an output paramter.", "name": "output_value",
"type": "String"}], "version": "print-01@sha256=d511ac628d43cc5b393fbebd10be93662b30117f1413b84afd4e7b2e5ff5ed33"}'
tekton.dev/template: ''
timeout: 525600m
- name: any-seq-0
taskSpec:
steps:
- name: main
args:
- --namespace
- $(params.pipelineRun-namespace)
- --prName
- $(params.pipelineRun-name)
- --taskList
- print-00,print-01
- --statusPath
- $(results.status.path)
command:
- any-task
image: dspipelines/any-sequencer:latest
results:
- name: status
description: /tmp/outputs/status/data
metadata:
labels:
pipelines.kubeflow.org/pipelinename: ''
pipelines.kubeflow.org/generation: ''
pipelines.kubeflow.org/cache_enabled: "true"
annotations:
pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-0", "outputs":
[{"description": "The output file to create the status", "name": "status"}],
"version": "any-seq-0@sha256=2a9bf729444af0ff906f5a6a77eff3eecfd419ebfbdb51e5f04cdfc11d58a7b8"}'
tekton.dev/template: ''
params:
- name: pipelineRun-name
- name: pipelineRun-namespace
timeout: 525600m
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: pipelineRun-namespace
value: $(context.pipelineRun.namespace)
- name: any-sequencer-looped-for-loop-1
taskRef:
apiVersion: custom.tekton.dev/v1alpha1
kind: PipelineLoop
name: any-sequencer-looped-for-loop-1
params:
- name: param-loop-item
value: $(params.param)
timeout: 525600m