kubeflow
/
kfp-tekton
mirror of https://github.com/kubeflow/kfp-tekton.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add artifact mid path to big data passing (#751) 

* add artifact mid path to big data passing

* add artifact mid path to big data passing

* address comments
This commit is contained in:
Tommy Li 2021-10-19 04:04:33 -07:00 committed by GitHub
parent e2468832da
commit f2fb08477c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 409 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
sdk/python/kfp_tekton/compiler/_data_passing_rewriter.py
View File

@ -15,6 +15,7 @@
import copy
import json
import re
import pathlib
from typing import List, Optional, Set
@ -486,7 +487,7 @@ def big_data_passing_tasks(prname: str, task: dict, pipelinerun_template: dict,
# $(workspaces.task_name.path)/task_name-task_output.get('name')
placeholder = '$(results.%s.path)' % (sanitize_k8s_name(
task_output.get('name')))
workspaces_parameter = '$(workspaces.%s.path)/%s-%s' % (
workspaces_parameter = '$(workspaces.%s.path)/artifacts/$(context.pipelineRun.name)/%s/%s' % (
task_name, task_name, task_output.get('name'))
task['taskSpec'] = replace_big_data_placeholder(
task.get("taskSpec", {}), placeholder, workspaces_parameter)
@ -512,8 +513,21 @@ def big_data_passing_tasks(prname: str, task: dict, pipelinerun_template: dict,
for task_artifact in task_artifacts:
if task_artifact.get('name') == task_param.get('name'):
placeholder = task_artifact.get('path')
workspaces_parameter = '$(workspaces.%s.path)/%s' % (
task_name, task_param.get('name'))
task_param_task_name = ""
task_param_param_name = ""
for o_task in outputs_tasks:
if '-'.join(o_task) == task_param.get('name'):
task_param_task_name = o_task[0]
task_param_param_name = o_task[1]
break
# If the param name is constructed with task_name-param_name,
# use the current task_name as the path prefix
if task_param_task_name:
workspaces_parameter = '$(workspaces.%s.path)/artifacts/$(context.pipelineRun.name)/%s/%s' % (
task_name, task_param_task_name, task_param_param_name)
else:
workspaces_parameter = '$(workspaces.%s.path)/artifacts/$(context.pipelineRun.name)/%s/%s' % (
task_name, task_name, task_param.get('name'))
task['taskSpec'] = replace_big_data_placeholder(
task_spec, placeholder, workspaces_parameter)
task_spec = task.get('taskSpec', {})
@ -580,9 +594,12 @@ def input_artifacts_tasks_pr_params(template: dict, artifact: dict) -> dict:
task_spec = template.get('taskSpec', {})
task_params = task_spec.get('params', [])
for task_param in task_params:
workspaces_parameter = '$(workspaces.%s.path)/%s' % (
task_name, task_param.get('name'))
# For pipeline parameter input artifacts, it will never come from another task because pipeline
# params are global parameters. Thus, task_name will always be the executing task name.
workspaces_parameter = '$(workspaces.%s.path)/artifacts/$(context.pipelineRun.name)/%s/%s' % (
task_name, task_name, task_param.get('name'))
if 'raw' in artifact:
copy_inputs_step['script'] += 'mkdir -p %s\n' % pathlib.Path(workspaces_parameter).parent
copy_inputs_step['script'] += 'echo -n "%s" > %s\n' % (
artifact['raw']['data'], workspaces_parameter)

97
sdk/python/tests/compiler/testdata/big_data_passing.yaml vendored
View File

@ -40,11 +40,11 @@ metadata:
tekton.dev/artifact_endpoint_scheme: http://
tekton.dev/artifact_items: '{"gen-params": [["Output", "$(results.output.path)"]],
"print-params": [], "print-text": [], "print-text-2": [], "print-text-3": [],
"print-text-4": [], "print-text-5": [], "repeat-line": [["output_text", "$(workspaces.repeat-line.path)/repeat-line-output_text"]],
"split-text-lines": [["even_lines", "$(workspaces.split-text-lines.path)/split-text-lines-even_lines"],
["odd_lines", "$(workspaces.split-text-lines.path)/split-text-lines-odd_lines"]],
"sum-numbers": [["Output", "$(workspaces.sum-numbers.path)/sum-numbers-Output"]],
"write-numbers": [["numbers", "$(workspaces.write-numbers.path)/write-numbers-numbers"]]}'
"print-text-4": [], "print-text-5": [], "repeat-line": [["output_text", "$(workspaces.repeat-line.path)/artifacts/$(context.pipelineRun.name)/repeat-line/output_text"]],
"split-text-lines": [["even_lines", "$(workspaces.split-text-lines.path)/artifacts/$(context.pipelineRun.name)/split-text-lines/even_lines"],
["odd_lines", "$(workspaces.split-text-lines.path)/artifacts/$(context.pipelineRun.name)/split-text-lines/odd_lines"]],
"sum-numbers": [["Output", "$(workspaces.sum-numbers.path)/artifacts/$(context.pipelineRun.name)/sum-numbers/Output"]],
"write-numbers": [["numbers", "$(workspaces.write-numbers.path)/artifacts/$(context.pipelineRun.name)/write-numbers/numbers"]]}'
sidecar.istio.io/inject: "false"
pipelines.kubeflow.org/pipeline_spec: '{"name": "Big data passing"}'
spec:
@ -60,7 +60,7 @@ spec:
- --count
- '5000'
- --output-text
- $(workspaces.repeat-line.path)/repeat-line-output_text
- $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text
command:
- sh
- -ec
@ -95,11 +95,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.repeat-line.path)/repeat-line-output_text | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.output-text.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.repeat-line.path)/repeat-line-output_text $(results.output-text.path)
cp $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text $(results.output-text.path)
fi
onError: continue
results:
@ -135,17 +135,22 @@ spec:
tekton.dev/template: ''
workspaces:
- name: repeat-line
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: repeat-line
workspace: big-data-passing
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text.path)/repeat-line-output_text
- $(workspaces.print-text.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text
command:
- sh
- -ec
@ -190,12 +195,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text
workspace: big-data-passing
runAfter:
- repeat-line
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: split-text-lines
taskSpec:
steps:
@ -219,9 +229,9 @@ spec:
- --source
- /tmp/inputs/source/data
- --odd-lines
- $(workspaces.split-text-lines.path)/split-text-lines-odd_lines
- $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines
- --even-lines
- $(workspaces.split-text-lines.path)/split-text-lines-even_lines
- $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines
command:
- sh
- -ec
@ -266,17 +276,17 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/split-text-lines-odd_lines | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.odd-lines.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.split-text-lines.path)/split-text-lines-odd_lines $(results.odd-lines.path)
cp $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines $(results.odd-lines.path)
fi
ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/split-text-lines-even_lines | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.even-lines.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.split-text-lines.path)/split-text-lines-even_lines $(results.even-lines.path)
cp $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines $(results.even-lines.path)
fi
onError: continue
results:
@ -322,17 +332,22 @@ spec:
volumes:
- name: source
emptyDir: {}
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: split-text-lines
workspace: big-data-passing
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text-2
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text-2.path)/split-text-lines-odd_lines
- $(workspaces.print-text-2.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines
command:
- sh
- -ec
@ -377,19 +392,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text-2
workspace: big-data-passing
runAfter:
- split-text-lines
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text-3
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text-3.path)/split-text-lines-even_lines
- $(workspaces.print-text-3.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines
command:
- sh
- -ec
@ -434,12 +454,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text-3
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text-3
workspace: big-data-passing
runAfter:
- split-text-lines
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: write-numbers
taskSpec:
steps:
@ -450,7 +475,7 @@ spec:
- --count
- '100000'
- --numbers
- $(workspaces.write-numbers.path)/write-numbers-numbers
- $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers
command:
- sh
- -ec
@ -485,11 +510,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.write-numbers.path)/write-numbers-numbers | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.numbers.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.write-numbers.path)/write-numbers-numbers $(results.numbers.path)
cp $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers $(results.numbers.path)
fi
onError: continue
results:
@ -525,17 +550,22 @@ spec:
tekton.dev/template: ''
workspaces:
- name: write-numbers
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: write-numbers
workspace: big-data-passing
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text-4
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text-4.path)/write-numbers-numbers
- $(workspaces.print-text-4.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers
command:
- sh
- -ec
@ -580,21 +610,26 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text-4
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text-4
workspace: big-data-passing
runAfter:
- write-numbers
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: sum-numbers
taskSpec:
steps:
- name: main
args:
- --numbers
- $(workspaces.sum-numbers.path)/write-numbers-numbers
- $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers
- '----output-paths'
- $(workspaces.sum-numbers.path)/sum-numbers-Output
- $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output
command:
- sh
- -ec
@ -648,11 +683,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.sum-numbers.path)/sum-numbers-Output | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.output.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.sum-numbers.path)/sum-numbers-Output $(results.output.path)
cp $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output $(results.output.path)
fi
onError: continue
results:
@ -687,19 +722,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: sum-numbers
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: sum-numbers
workspace: big-data-passing
runAfter:
- write-numbers
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text-5
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text-5.path)/sum-numbers-Output
- $(workspaces.print-text-5.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output
command:
- sh
- -ec
@ -744,12 +784,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text-5
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text-5
workspace: big-data-passing
runAfter:
- sum-numbers
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: gen-params
taskSpec:
steps:

97
sdk/python/tests/compiler/testdata/big_data_passing_noninlined.yaml vendored
View File

@ -40,11 +40,11 @@ metadata:
tekton.dev/artifact_endpoint_scheme: http://
tekton.dev/artifact_items: '{"gen-params": [["Output", "$(results.output.path)"]],
"print-params": [], "print-text": [], "print-text-2": [], "print-text-3": [],
"print-text-4": [], "print-text-5": [], "repeat-line": [["output_text", "$(workspaces.repeat-line.path)/repeat-line-output_text"]],
"split-text-lines": [["even_lines", "$(workspaces.split-text-lines.path)/split-text-lines-even_lines"],
["odd_lines", "$(workspaces.split-text-lines.path)/split-text-lines-odd_lines"]],
"sum-numbers": [["Output", "$(workspaces.sum-numbers.path)/sum-numbers-Output"]],
"write-numbers": [["numbers", "$(workspaces.write-numbers.path)/write-numbers-numbers"]]}'
"print-text-4": [], "print-text-5": [], "repeat-line": [["output_text", "$(workspaces.repeat-line.path)/artifacts/$(context.pipelineRun.name)/repeat-line/output_text"]],
"split-text-lines": [["even_lines", "$(workspaces.split-text-lines.path)/artifacts/$(context.pipelineRun.name)/split-text-lines/even_lines"],
["odd_lines", "$(workspaces.split-text-lines.path)/artifacts/$(context.pipelineRun.name)/split-text-lines/odd_lines"]],
"sum-numbers": [["Output", "$(workspaces.sum-numbers.path)/artifacts/$(context.pipelineRun.name)/sum-numbers/Output"]],
"write-numbers": [["numbers", "$(workspaces.write-numbers.path)/artifacts/$(context.pipelineRun.name)/write-numbers/numbers"]]}'
sidecar.istio.io/inject: "false"
pipelines.kubeflow.org/pipeline_spec: '{"name": "Big data passing"}'
spec:
@ -60,7 +60,7 @@ spec:
- --count
- '5000'
- --output-text
- $(workspaces.repeat-line.path)/repeat-line-output_text
- $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text
command:
- sh
- -ec
@ -95,11 +95,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.repeat-line.path)/repeat-line-output_text | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.output-text.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.repeat-line.path)/repeat-line-output_text $(results.output-text.path)
cp $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text $(results.output-text.path)
fi
onError: continue
results:
@ -135,17 +135,22 @@ spec:
tekton.dev/template: ''
workspaces:
- name: repeat-line
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: repeat-line
workspace: big-data-passing
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text.path)/repeat-line-output_text
- $(workspaces.print-text.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text
command:
- sh
- -ec
@ -190,12 +195,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text
workspace: big-data-passing
runAfter:
- repeat-line
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: split-text-lines
taskSpec:
steps:
@ -219,9 +229,9 @@ spec:
- --source
- /tmp/inputs/source/data
- --odd-lines
- $(workspaces.split-text-lines.path)/split-text-lines-odd_lines
- $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines
- --even-lines
- $(workspaces.split-text-lines.path)/split-text-lines-even_lines
- $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines
command:
- sh
- -ec
@ -266,17 +276,17 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/split-text-lines-odd_lines | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.odd-lines.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.split-text-lines.path)/split-text-lines-odd_lines $(results.odd-lines.path)
cp $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines $(results.odd-lines.path)
fi
ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/split-text-lines-even_lines | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.even-lines.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.split-text-lines.path)/split-text-lines-even_lines $(results.even-lines.path)
cp $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines $(results.even-lines.path)
fi
onError: continue
results:
@ -322,17 +332,22 @@ spec:
volumes:
- name: source
emptyDir: {}
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: split-text-lines
workspace: big-data-passing
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text-2
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text-2.path)/split-text-lines-odd_lines
- $(workspaces.print-text-2.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines
command:
- sh
- -ec
@ -377,19 +392,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text-2
workspace: big-data-passing
runAfter:
- split-text-lines
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text-3
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text-3.path)/split-text-lines-even_lines
- $(workspaces.print-text-3.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines
command:
- sh
- -ec
@ -434,12 +454,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text-3
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text-3
workspace: big-data-passing
runAfter:
- split-text-lines
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: write-numbers
taskSpec:
steps:
@ -450,7 +475,7 @@ spec:
- --count
- '100000'
- --numbers
- $(workspaces.write-numbers.path)/write-numbers-numbers
- $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers
command:
- sh
- -ec
@ -485,11 +510,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.write-numbers.path)/write-numbers-numbers | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.numbers.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.write-numbers.path)/write-numbers-numbers $(results.numbers.path)
cp $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers $(results.numbers.path)
fi
onError: continue
results:
@ -525,17 +550,22 @@ spec:
tekton.dev/template: ''
workspaces:
- name: write-numbers
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: write-numbers
workspace: big-data-passing
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text-4
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text-4.path)/write-numbers-numbers
- $(workspaces.print-text-4.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers
command:
- sh
- -ec
@ -580,21 +610,26 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text-4
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text-4
workspace: big-data-passing
runAfter:
- write-numbers
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: sum-numbers
taskSpec:
steps:
- name: main
args:
- --numbers
- $(workspaces.sum-numbers.path)/write-numbers-numbers
- $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers
- '----output-paths'
- $(workspaces.sum-numbers.path)/sum-numbers-Output
- $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output
command:
- sh
- -ec
@ -648,11 +683,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.sum-numbers.path)/sum-numbers-Output | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.output.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.sum-numbers.path)/sum-numbers-Output $(results.output.path)
cp $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output $(results.output.path)
fi
onError: continue
results:
@ -687,19 +722,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: sum-numbers
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: sum-numbers
workspace: big-data-passing
runAfter:
- write-numbers
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: print-text-5
taskSpec:
steps:
- name: main
args:
- --text
- $(workspaces.print-text-5.path)/sum-numbers-Output
- $(workspaces.print-text-5.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output
command:
- sh
- -ec
@ -744,12 +784,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: print-text-5
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: print-text-5
workspace: big-data-passing
runAfter:
- sum-numbers
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: gen-params
taskSpec:
steps:

44
sdk/python/tests/compiler/testdata/create_component_from_func.yaml vendored
View File

@ -32,10 +32,10 @@ metadata:
tekton.dev/artifact_bucket: mlpipeline
tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
tekton.dev/artifact_endpoint_scheme: http://
tekton.dev/artifact_items: '{"get-subdirectory": [["Subdir", "$(workspaces.get-subdirectory.path)/get-subdirectory-Subdir"]],
tekton.dev/artifact_items: '{"get-subdirectory": [["Subdir", "$(workspaces.get-subdirectory.path)/artifacts/$(context.pipelineRun.name)/get-subdirectory/Subdir"]],
"list-items": [["Items", "$(results.items.path)"]], "list-items-2": [["Items",
"$(results.items.path)"]], "produce-dir-with-files-python-op": [["output_dir",
"$(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir"]]}'
"$(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(context.pipelineRun.name)/produce-dir-with-files-python-op/output_dir"]]}'
sidecar.istio.io/inject: "false"
pipelines.kubeflow.org/pipeline_spec: '{"name": "create-component-from-function"}'
spec:
@ -49,7 +49,7 @@ spec:
- --num-files
- '15'
- --output-dir
- $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir
- $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir
command:
- sh
- -ec
@ -85,11 +85,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.output-dir.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir $(results.output-dir.path)
cp $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir $(results.output-dir.path)
fi
onError: continue
results:
@ -124,10 +124,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: produce-dir-with-files-python-op
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: produce-dir-with-files-python-op
workspace: create-component-from-function
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: get-subdirectory
taskSpec:
steps:
@ -139,9 +144,9 @@ spec:
- |
mkdir -p "$(dirname "$2")"
cp -r "$0/$1" "$2"
- $(workspaces.get-subdirectory.path)/produce-dir-with-files-python-op-output_dir
- $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir
- subdir
- $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir
- $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir
image: alpine
- image: busybox
name: copy-results-artifacts
@ -149,11 +154,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.subdir.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir $(results.subdir.path)
cp $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir $(results.subdir.path)
fi
onError: continue
results:
@ -175,12 +180,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: get-subdirectory
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: get-subdirectory
workspace: create-component-from-function
runAfter:
- produce-dir-with-files-python-op
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: list-items
taskSpec:
steps:
@ -193,7 +203,7 @@ spec:
mkdir -p "$(dirname "$1")"
#ls --almost-all --recursive "$0" > "$1"
ls -A -R "$0" > "$1"
- $(workspaces.list-items.path)/get-subdirectory-Subdir
- $(workspaces.list-items.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir
- $(results.items.path)
image: alpine
results:
@ -214,12 +224,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: list-items
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: list-items
workspace: create-component-from-function
runAfter:
- get-subdirectory
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: list-items-2
taskSpec:
steps:
@ -232,8 +247,8 @@ spec:
mkdir -p "$(dirname "$2")"
ls -A -R "$0" > "$2"
ls -A -R "$1" >> "$2"
- $(workspaces.list-items-2.path)/get-subdirectory-Subdir
- $(workspaces.list-items-2.path)/produce-dir-with-files-python-op-output_dir
- $(workspaces.list-items-2.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir
- $(workspaces.list-items-2.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir
- $(results.items.path)
image: alpine
results:
@ -256,6 +271,8 @@ spec:
tekton.dev/template: ''
workspaces:
- name: list-items-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: list-items-2
@ -263,6 +280,9 @@ spec:
runAfter:
- get-subdirectory
- produce-dir-with-files-python-op
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
workspaces:
- name: create-component-from-function
timeout: 0s

44
sdk/python/tests/compiler/testdata/create_component_from_func_noninlined.yaml vendored
View File

@ -32,10 +32,10 @@ metadata:
tekton.dev/artifact_bucket: mlpipeline
tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
tekton.dev/artifact_endpoint_scheme: http://
tekton.dev/artifact_items: '{"get-subdirectory": [["Subdir", "$(workspaces.get-subdirectory.path)/get-subdirectory-Subdir"]],
tekton.dev/artifact_items: '{"get-subdirectory": [["Subdir", "$(workspaces.get-subdirectory.path)/artifacts/$(context.pipelineRun.name)/get-subdirectory/Subdir"]],
"list-items": [["Items", "$(results.items.path)"]], "list-items-2": [["Items",
"$(results.items.path)"]], "produce-dir-with-files-python-op": [["output_dir",
"$(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir"]]}'
"$(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(context.pipelineRun.name)/produce-dir-with-files-python-op/output_dir"]]}'
sidecar.istio.io/inject: "false"
pipelines.kubeflow.org/pipeline_spec: '{"name": "create-component-from-function"}'
spec:
@ -49,7 +49,7 @@ spec:
- --num-files
- '15'
- --output-dir
- $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir
- $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir
command:
- sh
- -ec
@ -85,11 +85,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.output-dir.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir $(results.output-dir.path)
cp $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir $(results.output-dir.path)
fi
onError: continue
results:
@ -124,10 +124,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: produce-dir-with-files-python-op
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: produce-dir-with-files-python-op
workspace: create-component-from-function
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: get-subdirectory
taskSpec:
steps:
@ -139,9 +144,9 @@ spec:
- |
mkdir -p "$(dirname "$2")"
cp -r "$0/$1" "$2"
- $(workspaces.get-subdirectory.path)/produce-dir-with-files-python-op-output_dir
- $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir
- subdir
- $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir
- $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir
image: alpine
- image: busybox
name: copy-results-artifacts
@ -149,11 +154,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.subdir.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir $(results.subdir.path)
cp $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir $(results.subdir.path)
fi
onError: continue
results:
@ -175,12 +180,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: get-subdirectory
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: get-subdirectory
workspace: create-component-from-function
runAfter:
- produce-dir-with-files-python-op
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: list-items
taskSpec:
steps:
@ -193,7 +203,7 @@ spec:
mkdir -p "$(dirname "$1")"
#ls --almost-all --recursive "$0" > "$1"
ls -A -R "$0" > "$1"
- $(workspaces.list-items.path)/get-subdirectory-Subdir
- $(workspaces.list-items.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir
- $(results.items.path)
image: alpine
results:
@ -214,12 +224,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: list-items
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: list-items
workspace: create-component-from-function
runAfter:
- get-subdirectory
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: list-items-2
taskSpec:
steps:
@ -232,8 +247,8 @@ spec:
mkdir -p "$(dirname "$2")"
ls -A -R "$0" > "$2"
ls -A -R "$1" >> "$2"
- $(workspaces.list-items-2.path)/get-subdirectory-Subdir
- $(workspaces.list-items-2.path)/produce-dir-with-files-python-op-output_dir
- $(workspaces.list-items-2.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir
- $(workspaces.list-items-2.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir
- $(results.items.path)
image: alpine
results:
@ -256,6 +271,8 @@ spec:
tekton.dev/template: ''
workspaces:
- name: list-items-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: list-items-2
@ -263,6 +280,9 @@ spec:
runAfter:
- get-subdirectory
- produce-dir-with-files-python-op
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
workspaces:
- name: create-component-from-function
timeout: 0s

158
sdk/python/tests/compiler/testdata/data_passing_pipeline_complete.yaml vendored
View File

@ -52,9 +52,9 @@ metadata:
[], "consume-string-as-file-2": [], "consume-string-as-file-3": [], "consume-string-as-file-4":
[], "consume-string-as-file-5": [], "consume-string-as-value": [], "consume-string-as-value-2":
[], "consume-string-as-value-3": [], "consume-string-as-value-4": [], "consume-string-as-value-5":
[], "produce-anything": [["data", "$(workspaces.produce-anything.path)/produce-anything-data"]],
"produce-something": [["data", "$(workspaces.produce-something.path)/produce-something-data"]],
"produce-string": [["Output", "$(workspaces.produce-string.path)/produce-string-Output"]]}'
[], "produce-anything": [["data", "$(workspaces.produce-anything.path)/artifacts/$(context.pipelineRun.name)/produce-anything/data"]],
"produce-something": [["data", "$(workspaces.produce-something.path)/artifacts/$(context.pipelineRun.name)/produce-something/data"]],
"produce-string": [["Output", "$(workspaces.produce-string.path)/artifacts/$(context.pipelineRun.name)/produce-string/Output"]]}'
sidecar.istio.io/inject: "false"
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "anything_param",
"name": "anything_param", "optional": true}, {"default": "something_param",
@ -84,7 +84,7 @@ spec:
- name: main
args:
- --data
- $(workspaces.produce-anything.path)/produce-anything-data
- $(workspaces.produce-anything.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data
command:
- sh
- -ec
@ -115,11 +115,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.produce-anything.path)/produce-anything-data | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.produce-anything.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.data.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.produce-anything.path)/produce-anything-data $(results.data.path)
cp $(workspaces.produce-anything.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data $(results.data.path)
fi
onError: continue
results:
@ -146,17 +146,22 @@ spec:
tekton.dev/template: ''
workspaces:
- name: produce-anything
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: produce-anything
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: produce-something
taskSpec:
steps:
- name: main
args:
- --data
- $(workspaces.produce-something.path)/produce-something-data
- $(workspaces.produce-something.path)/artifacts/$(params.pipelineRun-name)/produce-something/data
command:
- sh
- -ec
@ -187,11 +192,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.produce-something.path)/produce-something-data | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.produce-something.path)/artifacts/$(params.pipelineRun-name)/produce-something/data | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.data.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.produce-something.path)/produce-something-data $(results.data.path)
cp $(workspaces.produce-something.path)/artifacts/$(params.pipelineRun-name)/produce-something/data $(results.data.path)
fi
onError: continue
results:
@ -218,17 +223,22 @@ spec:
tekton.dev/template: ''
workspaces:
- name: produce-something
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: produce-something
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: produce-string
taskSpec:
steps:
- name: main
args:
- '----output-paths'
- $(workspaces.produce-string.path)/produce-string-Output
- $(workspaces.produce-string.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output
command:
- sh
- -ec
@ -275,11 +285,11 @@ spec:
#!/bin/sh
set -exo pipefail
TOTAL_SIZE=0
ARTIFACT_SIZE=`wc -c $(workspaces.produce-string.path)/produce-string-Output | awk '{print $1}'`
ARTIFACT_SIZE=`wc -c $(workspaces.produce-string.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output | awk '{print $1}'`
TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE)
touch $(results.output.path)
if [[ $TOTAL_SIZE -lt 3072 ]]; then
cp $(workspaces.produce-string.path)/produce-string-Output $(results.output.path)
cp $(workspaces.produce-string.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output $(results.output.path)
fi
onError: continue
results:
@ -310,10 +320,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: produce-string
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: produce-string
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-anything-as-value
taskSpec:
steps:
@ -961,11 +976,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "anything_param" > $(workspaces.consume-anything-as-file-2.path)/anything_param
mkdir -p $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2
echo -n "anything_param" > $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2/anything_param
- name: main
args:
- --data
- $(workspaces.consume-anything-as-file-2.path)/anything_param
- $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2/anything_param
command:
- sh
- -ec
@ -1005,10 +1021,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-anything-as-file-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-anything-as-file-2
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-anything-as-file-3
taskSpec:
steps:
@ -1017,11 +1038,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "something_param" > $(workspaces.consume-anything-as-file-3.path)/something_param
mkdir -p $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3
echo -n "something_param" > $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3/something_param
- name: main
args:
- --data
- $(workspaces.consume-anything-as-file-3.path)/something_param
- $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3/something_param
command:
- sh
- -ec
@ -1061,10 +1083,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-anything-as-file-3
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-anything-as-file-3
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-anything-as-file-4
taskSpec:
steps:
@ -1073,11 +1100,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "string_param" > $(workspaces.consume-anything-as-file-4.path)/string_param
mkdir -p $(workspaces.consume-anything-as-file-4.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-4
echo -n "string_param" > $(workspaces.consume-anything-as-file-4.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-4/string_param
- name: main
args:
- --data
- $(workspaces.consume-anything-as-file-4.path)/string_param
- $(workspaces.consume-anything-as-file-4.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-4/string_param
command:
- sh
- -ec
@ -1117,10 +1145,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-anything-as-file-4
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-anything-as-file-4
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-something-as-file-2
taskSpec:
steps:
@ -1129,11 +1162,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "anything_param" > $(workspaces.consume-something-as-file-2.path)/anything_param
mkdir -p $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2
echo -n "anything_param" > $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2/anything_param
- name: main
args:
- --data
- $(workspaces.consume-something-as-file-2.path)/anything_param
- $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2/anything_param
command:
- sh
- -ec
@ -1173,10 +1207,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-something-as-file-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-something-as-file-2
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-something-as-file-3
taskSpec:
steps:
@ -1185,11 +1224,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "something_param" > $(workspaces.consume-something-as-file-3.path)/something_param
mkdir -p $(workspaces.consume-something-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-3
echo -n "something_param" > $(workspaces.consume-something-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-3/something_param
- name: main
args:
- --data
- $(workspaces.consume-something-as-file-3.path)/something_param
- $(workspaces.consume-something-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-3/something_param
command:
- sh
- -ec
@ -1229,10 +1269,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-something-as-file-3
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-something-as-file-3
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-string-as-file-2
taskSpec:
steps:
@ -1241,11 +1286,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "anything_param" > $(workspaces.consume-string-as-file-2.path)/anything_param
mkdir -p $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2
echo -n "anything_param" > $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2/anything_param
- name: main
args:
- --data
- $(workspaces.consume-string-as-file-2.path)/anything_param
- $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2/anything_param
command:
- sh
- -ec
@ -1285,10 +1331,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-string-as-file-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-string-as-file-2
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-string-as-file-3
taskSpec:
steps:
@ -1297,11 +1348,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "string_param" > $(workspaces.consume-string-as-file-3.path)/string_param
mkdir -p $(workspaces.consume-string-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-3
echo -n "string_param" > $(workspaces.consume-string-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-3/string_param
- name: main
args:
- --data
- $(workspaces.consume-string-as-file-3.path)/string_param
- $(workspaces.consume-string-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-3/string_param
command:
- sh
- -ec
@ -1341,10 +1393,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-string-as-file-3
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-string-as-file-3
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-anything-as-value-5
params:
- name: produce-anything-data
@ -1687,7 +1744,7 @@ spec:
- name: main
args:
- --data
- $(workspaces.consume-anything-as-file-5.path)/produce-anything-data
- $(workspaces.consume-anything-as-file-5.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data
command:
- sh
- -ec
@ -1727,19 +1784,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-anything-as-file-5
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-anything-as-file-5
workspace: data-passing-pipeline
runAfter:
- produce-anything
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-anything-as-file-6
taskSpec:
steps:
- name: main
args:
- --data
- $(workspaces.consume-anything-as-file-6.path)/produce-something-data
- $(workspaces.consume-anything-as-file-6.path)/artifacts/$(params.pipelineRun-name)/produce-something/data
command:
- sh
- -ec
@ -1779,19 +1841,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-anything-as-file-6
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-anything-as-file-6
workspace: data-passing-pipeline
runAfter:
- produce-something
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-anything-as-file-7
taskSpec:
steps:
- name: main
args:
- --data
- $(workspaces.consume-anything-as-file-7.path)/produce-string-Output
- $(workspaces.consume-anything-as-file-7.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output
command:
- sh
- -ec
@ -1831,19 +1898,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-anything-as-file-7
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-anything-as-file-7
workspace: data-passing-pipeline
runAfter:
- produce-string
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-something-as-file-4
taskSpec:
steps:
- name: main
args:
- --data
- $(workspaces.consume-something-as-file-4.path)/produce-anything-data
- $(workspaces.consume-something-as-file-4.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data
command:
- sh
- -ec
@ -1883,19 +1955,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-something-as-file-4
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-something-as-file-4
workspace: data-passing-pipeline
runAfter:
- produce-anything
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-something-as-file-5
taskSpec:
steps:
- name: main
args:
- --data
- $(workspaces.consume-something-as-file-5.path)/produce-something-data
- $(workspaces.consume-something-as-file-5.path)/artifacts/$(params.pipelineRun-name)/produce-something/data
command:
- sh
- -ec
@ -1935,19 +2012,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-something-as-file-5
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-something-as-file-5
workspace: data-passing-pipeline
runAfter:
- produce-something
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-string-as-file-4
taskSpec:
steps:
- name: main
args:
- --data
- $(workspaces.consume-string-as-file-4.path)/produce-anything-data
- $(workspaces.consume-string-as-file-4.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data
command:
- sh
- -ec
@ -1987,19 +2069,24 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-string-as-file-4
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-string-as-file-4
workspace: data-passing-pipeline
runAfter:
- produce-anything
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-string-as-file-5
taskSpec:
steps:
- name: main
args:
- --data
- $(workspaces.consume-string-as-file-5.path)/produce-string-Output
- $(workspaces.consume-string-as-file-5.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output
command:
- sh
- -ec
@ -2039,12 +2126,17 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-string-as-file-5
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-string-as-file-5
workspace: data-passing-pipeline
runAfter:
- produce-string
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
workspaces:
- name: data-passing-pipeline
timeout: 0s

70
sdk/python/tests/compiler/testdata/data_passing_pipeline_param_as_file.yaml vendored
View File

@ -56,11 +56,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "anything_param" > $(workspaces.consume-anything-as-file.path)/anything_param
mkdir -p $(workspaces.consume-anything-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file
echo -n "anything_param" > $(workspaces.consume-anything-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file/anything_param
- name: main
args:
- --data
- $(workspaces.consume-anything-as-file.path)/anything_param
- $(workspaces.consume-anything-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file/anything_param
command:
- sh
- -ec
@ -100,10 +101,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-anything-as-file
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-anything-as-file
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-anything-as-file-2
taskSpec:
steps:
@ -112,11 +118,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "something_param" > $(workspaces.consume-anything-as-file-2.path)/something_param
mkdir -p $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2
echo -n "something_param" > $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2/something_param
- name: main
args:
- --data
- $(workspaces.consume-anything-as-file-2.path)/something_param
- $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2/something_param
command:
- sh
- -ec
@ -156,10 +163,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-anything-as-file-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-anything-as-file-2
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-anything-as-file-3
taskSpec:
steps:
@ -168,11 +180,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "string_param" > $(workspaces.consume-anything-as-file-3.path)/string_param
mkdir -p $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3
echo -n "string_param" > $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3/string_param
- name: main
args:
- --data
- $(workspaces.consume-anything-as-file-3.path)/string_param
- $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3/string_param
command:
- sh
- -ec
@ -212,10 +225,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-anything-as-file-3
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-anything-as-file-3
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-something-as-file
taskSpec:
steps:
@ -224,11 +242,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "anything_param" > $(workspaces.consume-something-as-file.path)/anything_param
mkdir -p $(workspaces.consume-something-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file
echo -n "anything_param" > $(workspaces.consume-something-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file/anything_param
- name: main
args:
- --data
- $(workspaces.consume-something-as-file.path)/anything_param
- $(workspaces.consume-something-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file/anything_param
command:
- sh
- -ec
@ -268,10 +287,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-something-as-file
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-something-as-file
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-something-as-file-2
taskSpec:
steps:
@ -280,11 +304,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "something_param" > $(workspaces.consume-something-as-file-2.path)/something_param
mkdir -p $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2
echo -n "something_param" > $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2/something_param
- name: main
args:
- --data
- $(workspaces.consume-something-as-file-2.path)/something_param
- $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2/something_param
command:
- sh
- -ec
@ -324,10 +349,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-something-as-file-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-something-as-file-2
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-string-as-file
taskSpec:
steps:
@ -336,11 +366,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "anything_param" > $(workspaces.consume-string-as-file.path)/anything_param
mkdir -p $(workspaces.consume-string-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file
echo -n "anything_param" > $(workspaces.consume-string-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file/anything_param
- name: main
args:
- --data
- $(workspaces.consume-string-as-file.path)/anything_param
- $(workspaces.consume-string-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file/anything_param
command:
- sh
- -ec
@ -380,10 +411,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-string-as-file
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-string-as-file
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
- name: consume-string-as-file-2
taskSpec:
steps:
@ -392,11 +428,12 @@ spec:
script: |
#!/bin/sh
set -exo pipefail
echo -n "string_param" > $(workspaces.consume-string-as-file-2.path)/string_param
mkdir -p $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2
echo -n "string_param" > $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2/string_param
- name: main
args:
- --data
- $(workspaces.consume-string-as-file-2.path)/string_param
- $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2/string_param
command:
- sh
- -ec
@ -436,10 +473,15 @@ spec:
tekton.dev/template: ''
workspaces:
- name: consume-string-as-file-2
params:
- name: pipelineRun-name
timeout: 0s
workspaces:
- name: consume-string-as-file-2
workspace: data-passing-pipeline
params:
- name: pipelineRun-name
value: $(context.pipelineRun.name)
workspaces:
- name: data-passing-pipeline
timeout: 0s