diff --git a/sdk/python/kfp_tekton/compiler/_data_passing_rewriter.py b/sdk/python/kfp_tekton/compiler/_data_passing_rewriter.py index ac79d7b39..472be7840 100644 --- a/sdk/python/kfp_tekton/compiler/_data_passing_rewriter.py +++ b/sdk/python/kfp_tekton/compiler/_data_passing_rewriter.py @@ -15,6 +15,7 @@ import copy import json import re +import pathlib from typing import List, Optional, Set @@ -486,7 +487,7 @@ def big_data_passing_tasks(prname: str, task: dict, pipelinerun_template: dict, # $(workspaces.task_name.path)/task_name-task_output.get('name') placeholder = '$(results.%s.path)' % (sanitize_k8s_name( task_output.get('name'))) - workspaces_parameter = '$(workspaces.%s.path)/%s-%s' % ( + workspaces_parameter = '$(workspaces.%s.path)/artifacts/$(context.pipelineRun.name)/%s/%s' % ( task_name, task_name, task_output.get('name')) task['taskSpec'] = replace_big_data_placeholder( task.get("taskSpec", {}), placeholder, workspaces_parameter) @@ -512,8 +513,21 @@ def big_data_passing_tasks(prname: str, task: dict, pipelinerun_template: dict, for task_artifact in task_artifacts: if task_artifact.get('name') == task_param.get('name'): placeholder = task_artifact.get('path') - workspaces_parameter = '$(workspaces.%s.path)/%s' % ( - task_name, task_param.get('name')) + task_param_task_name = "" + task_param_param_name = "" + for o_task in outputs_tasks: + if '-'.join(o_task) == task_param.get('name'): + task_param_task_name = o_task[0] + task_param_param_name = o_task[1] + break + # If the param name is constructed with task_name-param_name, + # use the current task_name as the path prefix + if task_param_task_name: + workspaces_parameter = '$(workspaces.%s.path)/artifacts/$(context.pipelineRun.name)/%s/%s' % ( + task_name, task_param_task_name, task_param_param_name) + else: + workspaces_parameter = '$(workspaces.%s.path)/artifacts/$(context.pipelineRun.name)/%s/%s' % ( + task_name, task_name, task_param.get('name')) task['taskSpec'] = replace_big_data_placeholder( task_spec, placeholder, workspaces_parameter) task_spec = task.get('taskSpec', {}) @@ -580,9 +594,12 @@ def input_artifacts_tasks_pr_params(template: dict, artifact: dict) -> dict: task_spec = template.get('taskSpec', {}) task_params = task_spec.get('params', []) for task_param in task_params: - workspaces_parameter = '$(workspaces.%s.path)/%s' % ( - task_name, task_param.get('name')) + # For pipeline parameter input artifacts, it will never come from another task because pipeline + # params are global parameters. Thus, task_name will always be the executing task name. + workspaces_parameter = '$(workspaces.%s.path)/artifacts/$(context.pipelineRun.name)/%s/%s' % ( + task_name, task_name, task_param.get('name')) if 'raw' in artifact: + copy_inputs_step['script'] += 'mkdir -p %s\n' % pathlib.Path(workspaces_parameter).parent copy_inputs_step['script'] += 'echo -n "%s" > %s\n' % ( artifact['raw']['data'], workspaces_parameter) diff --git a/sdk/python/tests/compiler/testdata/big_data_passing.yaml b/sdk/python/tests/compiler/testdata/big_data_passing.yaml index c952dda8d..395fe192a 100644 --- a/sdk/python/tests/compiler/testdata/big_data_passing.yaml +++ b/sdk/python/tests/compiler/testdata/big_data_passing.yaml @@ -40,11 +40,11 @@ metadata: tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/artifact_items: '{"gen-params": [["Output", "$(results.output.path)"]], "print-params": [], "print-text": [], "print-text-2": [], "print-text-3": [], - "print-text-4": [], "print-text-5": [], "repeat-line": [["output_text", "$(workspaces.repeat-line.path)/repeat-line-output_text"]], - "split-text-lines": [["even_lines", "$(workspaces.split-text-lines.path)/split-text-lines-even_lines"], - ["odd_lines", "$(workspaces.split-text-lines.path)/split-text-lines-odd_lines"]], - "sum-numbers": [["Output", "$(workspaces.sum-numbers.path)/sum-numbers-Output"]], - "write-numbers": [["numbers", "$(workspaces.write-numbers.path)/write-numbers-numbers"]]}' + "print-text-4": [], "print-text-5": [], "repeat-line": [["output_text", "$(workspaces.repeat-line.path)/artifacts/$(context.pipelineRun.name)/repeat-line/output_text"]], + "split-text-lines": [["even_lines", "$(workspaces.split-text-lines.path)/artifacts/$(context.pipelineRun.name)/split-text-lines/even_lines"], + ["odd_lines", "$(workspaces.split-text-lines.path)/artifacts/$(context.pipelineRun.name)/split-text-lines/odd_lines"]], + "sum-numbers": [["Output", "$(workspaces.sum-numbers.path)/artifacts/$(context.pipelineRun.name)/sum-numbers/Output"]], + "write-numbers": [["numbers", "$(workspaces.write-numbers.path)/artifacts/$(context.pipelineRun.name)/write-numbers/numbers"]]}' sidecar.istio.io/inject: "false" pipelines.kubeflow.org/pipeline_spec: '{"name": "Big data passing"}' spec: @@ -60,7 +60,7 @@ spec: - --count - '5000' - --output-text - - $(workspaces.repeat-line.path)/repeat-line-output_text + - $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text command: - sh - -ec @@ -95,11 +95,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.repeat-line.path)/repeat-line-output_text | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.output-text.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.repeat-line.path)/repeat-line-output_text $(results.output-text.path) + cp $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text $(results.output-text.path) fi onError: continue results: @@ -135,17 +135,22 @@ spec: tekton.dev/template: '' workspaces: - name: repeat-line + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: repeat-line workspace: big-data-passing + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text.path)/repeat-line-output_text + - $(workspaces.print-text.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text command: - sh - -ec @@ -190,12 +195,17 @@ spec: tekton.dev/template: '' workspaces: - name: print-text + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text workspace: big-data-passing runAfter: - repeat-line + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: split-text-lines taskSpec: steps: @@ -219,9 +229,9 @@ spec: - --source - /tmp/inputs/source/data - --odd-lines - - $(workspaces.split-text-lines.path)/split-text-lines-odd_lines + - $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines - --even-lines - - $(workspaces.split-text-lines.path)/split-text-lines-even_lines + - $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines command: - sh - -ec @@ -266,17 +276,17 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/split-text-lines-odd_lines | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.odd-lines.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.split-text-lines.path)/split-text-lines-odd_lines $(results.odd-lines.path) + cp $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines $(results.odd-lines.path) fi - ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/split-text-lines-even_lines | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.even-lines.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.split-text-lines.path)/split-text-lines-even_lines $(results.even-lines.path) + cp $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines $(results.even-lines.path) fi onError: continue results: @@ -322,17 +332,22 @@ spec: volumes: - name: source emptyDir: {} + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: split-text-lines workspace: big-data-passing + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text-2 taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text-2.path)/split-text-lines-odd_lines + - $(workspaces.print-text-2.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines command: - sh - -ec @@ -377,19 +392,24 @@ spec: tekton.dev/template: '' workspaces: - name: print-text-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text-2 workspace: big-data-passing runAfter: - split-text-lines + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text-3 taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text-3.path)/split-text-lines-even_lines + - $(workspaces.print-text-3.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines command: - sh - -ec @@ -434,12 +454,17 @@ spec: tekton.dev/template: '' workspaces: - name: print-text-3 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text-3 workspace: big-data-passing runAfter: - split-text-lines + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: write-numbers taskSpec: steps: @@ -450,7 +475,7 @@ spec: - --count - '100000' - --numbers - - $(workspaces.write-numbers.path)/write-numbers-numbers + - $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers command: - sh - -ec @@ -485,11 +510,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.write-numbers.path)/write-numbers-numbers | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.numbers.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.write-numbers.path)/write-numbers-numbers $(results.numbers.path) + cp $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers $(results.numbers.path) fi onError: continue results: @@ -525,17 +550,22 @@ spec: tekton.dev/template: '' workspaces: - name: write-numbers + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: write-numbers workspace: big-data-passing + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text-4 taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text-4.path)/write-numbers-numbers + - $(workspaces.print-text-4.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers command: - sh - -ec @@ -580,21 +610,26 @@ spec: tekton.dev/template: '' workspaces: - name: print-text-4 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text-4 workspace: big-data-passing runAfter: - write-numbers + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: sum-numbers taskSpec: steps: - name: main args: - --numbers - - $(workspaces.sum-numbers.path)/write-numbers-numbers + - $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers - '----output-paths' - - $(workspaces.sum-numbers.path)/sum-numbers-Output + - $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output command: - sh - -ec @@ -648,11 +683,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.sum-numbers.path)/sum-numbers-Output | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.output.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.sum-numbers.path)/sum-numbers-Output $(results.output.path) + cp $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output $(results.output.path) fi onError: continue results: @@ -687,19 +722,24 @@ spec: tekton.dev/template: '' workspaces: - name: sum-numbers + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: sum-numbers workspace: big-data-passing runAfter: - write-numbers + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text-5 taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text-5.path)/sum-numbers-Output + - $(workspaces.print-text-5.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output command: - sh - -ec @@ -744,12 +784,17 @@ spec: tekton.dev/template: '' workspaces: - name: print-text-5 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text-5 workspace: big-data-passing runAfter: - sum-numbers + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: gen-params taskSpec: steps: diff --git a/sdk/python/tests/compiler/testdata/big_data_passing_noninlined.yaml b/sdk/python/tests/compiler/testdata/big_data_passing_noninlined.yaml index c952dda8d..395fe192a 100644 --- a/sdk/python/tests/compiler/testdata/big_data_passing_noninlined.yaml +++ b/sdk/python/tests/compiler/testdata/big_data_passing_noninlined.yaml @@ -40,11 +40,11 @@ metadata: tekton.dev/artifact_endpoint_scheme: http:// tekton.dev/artifact_items: '{"gen-params": [["Output", "$(results.output.path)"]], "print-params": [], "print-text": [], "print-text-2": [], "print-text-3": [], - "print-text-4": [], "print-text-5": [], "repeat-line": [["output_text", "$(workspaces.repeat-line.path)/repeat-line-output_text"]], - "split-text-lines": [["even_lines", "$(workspaces.split-text-lines.path)/split-text-lines-even_lines"], - ["odd_lines", "$(workspaces.split-text-lines.path)/split-text-lines-odd_lines"]], - "sum-numbers": [["Output", "$(workspaces.sum-numbers.path)/sum-numbers-Output"]], - "write-numbers": [["numbers", "$(workspaces.write-numbers.path)/write-numbers-numbers"]]}' + "print-text-4": [], "print-text-5": [], "repeat-line": [["output_text", "$(workspaces.repeat-line.path)/artifacts/$(context.pipelineRun.name)/repeat-line/output_text"]], + "split-text-lines": [["even_lines", "$(workspaces.split-text-lines.path)/artifacts/$(context.pipelineRun.name)/split-text-lines/even_lines"], + ["odd_lines", "$(workspaces.split-text-lines.path)/artifacts/$(context.pipelineRun.name)/split-text-lines/odd_lines"]], + "sum-numbers": [["Output", "$(workspaces.sum-numbers.path)/artifacts/$(context.pipelineRun.name)/sum-numbers/Output"]], + "write-numbers": [["numbers", "$(workspaces.write-numbers.path)/artifacts/$(context.pipelineRun.name)/write-numbers/numbers"]]}' sidecar.istio.io/inject: "false" pipelines.kubeflow.org/pipeline_spec: '{"name": "Big data passing"}' spec: @@ -60,7 +60,7 @@ spec: - --count - '5000' - --output-text - - $(workspaces.repeat-line.path)/repeat-line-output_text + - $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text command: - sh - -ec @@ -95,11 +95,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.repeat-line.path)/repeat-line-output_text | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.output-text.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.repeat-line.path)/repeat-line-output_text $(results.output-text.path) + cp $(workspaces.repeat-line.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text $(results.output-text.path) fi onError: continue results: @@ -135,17 +135,22 @@ spec: tekton.dev/template: '' workspaces: - name: repeat-line + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: repeat-line workspace: big-data-passing + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text.path)/repeat-line-output_text + - $(workspaces.print-text.path)/artifacts/$(params.pipelineRun-name)/repeat-line/output_text command: - sh - -ec @@ -190,12 +195,17 @@ spec: tekton.dev/template: '' workspaces: - name: print-text + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text workspace: big-data-passing runAfter: - repeat-line + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: split-text-lines taskSpec: steps: @@ -219,9 +229,9 @@ spec: - --source - /tmp/inputs/source/data - --odd-lines - - $(workspaces.split-text-lines.path)/split-text-lines-odd_lines + - $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines - --even-lines - - $(workspaces.split-text-lines.path)/split-text-lines-even_lines + - $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines command: - sh - -ec @@ -266,17 +276,17 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/split-text-lines-odd_lines | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.odd-lines.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.split-text-lines.path)/split-text-lines-odd_lines $(results.odd-lines.path) + cp $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines $(results.odd-lines.path) fi - ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/split-text-lines-even_lines | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.even-lines.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.split-text-lines.path)/split-text-lines-even_lines $(results.even-lines.path) + cp $(workspaces.split-text-lines.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines $(results.even-lines.path) fi onError: continue results: @@ -322,17 +332,22 @@ spec: volumes: - name: source emptyDir: {} + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: split-text-lines workspace: big-data-passing + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text-2 taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text-2.path)/split-text-lines-odd_lines + - $(workspaces.print-text-2.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/odd_lines command: - sh - -ec @@ -377,19 +392,24 @@ spec: tekton.dev/template: '' workspaces: - name: print-text-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text-2 workspace: big-data-passing runAfter: - split-text-lines + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text-3 taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text-3.path)/split-text-lines-even_lines + - $(workspaces.print-text-3.path)/artifacts/$(params.pipelineRun-name)/split-text-lines/even_lines command: - sh - -ec @@ -434,12 +454,17 @@ spec: tekton.dev/template: '' workspaces: - name: print-text-3 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text-3 workspace: big-data-passing runAfter: - split-text-lines + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: write-numbers taskSpec: steps: @@ -450,7 +475,7 @@ spec: - --count - '100000' - --numbers - - $(workspaces.write-numbers.path)/write-numbers-numbers + - $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers command: - sh - -ec @@ -485,11 +510,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.write-numbers.path)/write-numbers-numbers | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.numbers.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.write-numbers.path)/write-numbers-numbers $(results.numbers.path) + cp $(workspaces.write-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers $(results.numbers.path) fi onError: continue results: @@ -525,17 +550,22 @@ spec: tekton.dev/template: '' workspaces: - name: write-numbers + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: write-numbers workspace: big-data-passing + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text-4 taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text-4.path)/write-numbers-numbers + - $(workspaces.print-text-4.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers command: - sh - -ec @@ -580,21 +610,26 @@ spec: tekton.dev/template: '' workspaces: - name: print-text-4 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text-4 workspace: big-data-passing runAfter: - write-numbers + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: sum-numbers taskSpec: steps: - name: main args: - --numbers - - $(workspaces.sum-numbers.path)/write-numbers-numbers + - $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/write-numbers/numbers - '----output-paths' - - $(workspaces.sum-numbers.path)/sum-numbers-Output + - $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output command: - sh - -ec @@ -648,11 +683,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.sum-numbers.path)/sum-numbers-Output | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.output.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.sum-numbers.path)/sum-numbers-Output $(results.output.path) + cp $(workspaces.sum-numbers.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output $(results.output.path) fi onError: continue results: @@ -687,19 +722,24 @@ spec: tekton.dev/template: '' workspaces: - name: sum-numbers + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: sum-numbers workspace: big-data-passing runAfter: - write-numbers + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: print-text-5 taskSpec: steps: - name: main args: - --text - - $(workspaces.print-text-5.path)/sum-numbers-Output + - $(workspaces.print-text-5.path)/artifacts/$(params.pipelineRun-name)/sum-numbers/Output command: - sh - -ec @@ -744,12 +784,17 @@ spec: tekton.dev/template: '' workspaces: - name: print-text-5 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: print-text-5 workspace: big-data-passing runAfter: - sum-numbers + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: gen-params taskSpec: steps: diff --git a/sdk/python/tests/compiler/testdata/create_component_from_func.yaml b/sdk/python/tests/compiler/testdata/create_component_from_func.yaml index bb4e069ea..668022afd 100644 --- a/sdk/python/tests/compiler/testdata/create_component_from_func.yaml +++ b/sdk/python/tests/compiler/testdata/create_component_from_func.yaml @@ -32,10 +32,10 @@ metadata: tekton.dev/artifact_bucket: mlpipeline tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 tekton.dev/artifact_endpoint_scheme: http:// - tekton.dev/artifact_items: '{"get-subdirectory": [["Subdir", "$(workspaces.get-subdirectory.path)/get-subdirectory-Subdir"]], + tekton.dev/artifact_items: '{"get-subdirectory": [["Subdir", "$(workspaces.get-subdirectory.path)/artifacts/$(context.pipelineRun.name)/get-subdirectory/Subdir"]], "list-items": [["Items", "$(results.items.path)"]], "list-items-2": [["Items", "$(results.items.path)"]], "produce-dir-with-files-python-op": [["output_dir", - "$(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir"]]}' + "$(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(context.pipelineRun.name)/produce-dir-with-files-python-op/output_dir"]]}' sidecar.istio.io/inject: "false" pipelines.kubeflow.org/pipeline_spec: '{"name": "create-component-from-function"}' spec: @@ -49,7 +49,7 @@ spec: - --num-files - '15' - --output-dir - - $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir + - $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir command: - sh - -ec @@ -85,11 +85,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.output-dir.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir $(results.output-dir.path) + cp $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir $(results.output-dir.path) fi onError: continue results: @@ -124,10 +124,15 @@ spec: tekton.dev/template: '' workspaces: - name: produce-dir-with-files-python-op + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: produce-dir-with-files-python-op workspace: create-component-from-function + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: get-subdirectory taskSpec: steps: @@ -139,9 +144,9 @@ spec: - | mkdir -p "$(dirname "$2")" cp -r "$0/$1" "$2" - - $(workspaces.get-subdirectory.path)/produce-dir-with-files-python-op-output_dir + - $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir - subdir - - $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir + - $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir image: alpine - image: busybox name: copy-results-artifacts @@ -149,11 +154,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.subdir.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir $(results.subdir.path) + cp $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir $(results.subdir.path) fi onError: continue results: @@ -175,12 +180,17 @@ spec: tekton.dev/template: '' workspaces: - name: get-subdirectory + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: get-subdirectory workspace: create-component-from-function runAfter: - produce-dir-with-files-python-op + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: list-items taskSpec: steps: @@ -193,7 +203,7 @@ spec: mkdir -p "$(dirname "$1")" #ls --almost-all --recursive "$0" > "$1" ls -A -R "$0" > "$1" - - $(workspaces.list-items.path)/get-subdirectory-Subdir + - $(workspaces.list-items.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir - $(results.items.path) image: alpine results: @@ -214,12 +224,17 @@ spec: tekton.dev/template: '' workspaces: - name: list-items + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: list-items workspace: create-component-from-function runAfter: - get-subdirectory + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: list-items-2 taskSpec: steps: @@ -232,8 +247,8 @@ spec: mkdir -p "$(dirname "$2")" ls -A -R "$0" > "$2" ls -A -R "$1" >> "$2" - - $(workspaces.list-items-2.path)/get-subdirectory-Subdir - - $(workspaces.list-items-2.path)/produce-dir-with-files-python-op-output_dir + - $(workspaces.list-items-2.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir + - $(workspaces.list-items-2.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir - $(results.items.path) image: alpine results: @@ -256,6 +271,8 @@ spec: tekton.dev/template: '' workspaces: - name: list-items-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: list-items-2 @@ -263,6 +280,9 @@ spec: runAfter: - get-subdirectory - produce-dir-with-files-python-op + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) workspaces: - name: create-component-from-function timeout: 0s diff --git a/sdk/python/tests/compiler/testdata/create_component_from_func_noninlined.yaml b/sdk/python/tests/compiler/testdata/create_component_from_func_noninlined.yaml index bb4e069ea..668022afd 100644 --- a/sdk/python/tests/compiler/testdata/create_component_from_func_noninlined.yaml +++ b/sdk/python/tests/compiler/testdata/create_component_from_func_noninlined.yaml @@ -32,10 +32,10 @@ metadata: tekton.dev/artifact_bucket: mlpipeline tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 tekton.dev/artifact_endpoint_scheme: http:// - tekton.dev/artifact_items: '{"get-subdirectory": [["Subdir", "$(workspaces.get-subdirectory.path)/get-subdirectory-Subdir"]], + tekton.dev/artifact_items: '{"get-subdirectory": [["Subdir", "$(workspaces.get-subdirectory.path)/artifacts/$(context.pipelineRun.name)/get-subdirectory/Subdir"]], "list-items": [["Items", "$(results.items.path)"]], "list-items-2": [["Items", "$(results.items.path)"]], "produce-dir-with-files-python-op": [["output_dir", - "$(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir"]]}' + "$(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(context.pipelineRun.name)/produce-dir-with-files-python-op/output_dir"]]}' sidecar.istio.io/inject: "false" pipelines.kubeflow.org/pipeline_spec: '{"name": "create-component-from-function"}' spec: @@ -49,7 +49,7 @@ spec: - --num-files - '15' - --output-dir - - $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir + - $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir command: - sh - -ec @@ -85,11 +85,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.output-dir.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.produce-dir-with-files-python-op.path)/produce-dir-with-files-python-op-output_dir $(results.output-dir.path) + cp $(workspaces.produce-dir-with-files-python-op.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir $(results.output-dir.path) fi onError: continue results: @@ -124,10 +124,15 @@ spec: tekton.dev/template: '' workspaces: - name: produce-dir-with-files-python-op + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: produce-dir-with-files-python-op workspace: create-component-from-function + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: get-subdirectory taskSpec: steps: @@ -139,9 +144,9 @@ spec: - | mkdir -p "$(dirname "$2")" cp -r "$0/$1" "$2" - - $(workspaces.get-subdirectory.path)/produce-dir-with-files-python-op-output_dir + - $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir - subdir - - $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir + - $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir image: alpine - image: busybox name: copy-results-artifacts @@ -149,11 +154,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.subdir.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.get-subdirectory.path)/get-subdirectory-Subdir $(results.subdir.path) + cp $(workspaces.get-subdirectory.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir $(results.subdir.path) fi onError: continue results: @@ -175,12 +180,17 @@ spec: tekton.dev/template: '' workspaces: - name: get-subdirectory + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: get-subdirectory workspace: create-component-from-function runAfter: - produce-dir-with-files-python-op + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: list-items taskSpec: steps: @@ -193,7 +203,7 @@ spec: mkdir -p "$(dirname "$1")" #ls --almost-all --recursive "$0" > "$1" ls -A -R "$0" > "$1" - - $(workspaces.list-items.path)/get-subdirectory-Subdir + - $(workspaces.list-items.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir - $(results.items.path) image: alpine results: @@ -214,12 +224,17 @@ spec: tekton.dev/template: '' workspaces: - name: list-items + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: list-items workspace: create-component-from-function runAfter: - get-subdirectory + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: list-items-2 taskSpec: steps: @@ -232,8 +247,8 @@ spec: mkdir -p "$(dirname "$2")" ls -A -R "$0" > "$2" ls -A -R "$1" >> "$2" - - $(workspaces.list-items-2.path)/get-subdirectory-Subdir - - $(workspaces.list-items-2.path)/produce-dir-with-files-python-op-output_dir + - $(workspaces.list-items-2.path)/artifacts/$(params.pipelineRun-name)/get-subdirectory/Subdir + - $(workspaces.list-items-2.path)/artifacts/$(params.pipelineRun-name)/produce-dir-with-files-python-op/output_dir - $(results.items.path) image: alpine results: @@ -256,6 +271,8 @@ spec: tekton.dev/template: '' workspaces: - name: list-items-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: list-items-2 @@ -263,6 +280,9 @@ spec: runAfter: - get-subdirectory - produce-dir-with-files-python-op + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) workspaces: - name: create-component-from-function timeout: 0s diff --git a/sdk/python/tests/compiler/testdata/data_passing_pipeline_complete.yaml b/sdk/python/tests/compiler/testdata/data_passing_pipeline_complete.yaml index 9e0b482b9..25c31d4d2 100644 --- a/sdk/python/tests/compiler/testdata/data_passing_pipeline_complete.yaml +++ b/sdk/python/tests/compiler/testdata/data_passing_pipeline_complete.yaml @@ -52,9 +52,9 @@ metadata: [], "consume-string-as-file-2": [], "consume-string-as-file-3": [], "consume-string-as-file-4": [], "consume-string-as-file-5": [], "consume-string-as-value": [], "consume-string-as-value-2": [], "consume-string-as-value-3": [], "consume-string-as-value-4": [], "consume-string-as-value-5": - [], "produce-anything": [["data", "$(workspaces.produce-anything.path)/produce-anything-data"]], - "produce-something": [["data", "$(workspaces.produce-something.path)/produce-something-data"]], - "produce-string": [["Output", "$(workspaces.produce-string.path)/produce-string-Output"]]}' + [], "produce-anything": [["data", "$(workspaces.produce-anything.path)/artifacts/$(context.pipelineRun.name)/produce-anything/data"]], + "produce-something": [["data", "$(workspaces.produce-something.path)/artifacts/$(context.pipelineRun.name)/produce-something/data"]], + "produce-string": [["Output", "$(workspaces.produce-string.path)/artifacts/$(context.pipelineRun.name)/produce-string/Output"]]}' sidecar.istio.io/inject: "false" pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "anything_param", "name": "anything_param", "optional": true}, {"default": "something_param", @@ -84,7 +84,7 @@ spec: - name: main args: - --data - - $(workspaces.produce-anything.path)/produce-anything-data + - $(workspaces.produce-anything.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data command: - sh - -ec @@ -115,11 +115,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.produce-anything.path)/produce-anything-data | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.produce-anything.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.data.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.produce-anything.path)/produce-anything-data $(results.data.path) + cp $(workspaces.produce-anything.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data $(results.data.path) fi onError: continue results: @@ -146,17 +146,22 @@ spec: tekton.dev/template: '' workspaces: - name: produce-anything + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: produce-anything workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: produce-something taskSpec: steps: - name: main args: - --data - - $(workspaces.produce-something.path)/produce-something-data + - $(workspaces.produce-something.path)/artifacts/$(params.pipelineRun-name)/produce-something/data command: - sh - -ec @@ -187,11 +192,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.produce-something.path)/produce-something-data | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.produce-something.path)/artifacts/$(params.pipelineRun-name)/produce-something/data | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.data.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.produce-something.path)/produce-something-data $(results.data.path) + cp $(workspaces.produce-something.path)/artifacts/$(params.pipelineRun-name)/produce-something/data $(results.data.path) fi onError: continue results: @@ -218,17 +223,22 @@ spec: tekton.dev/template: '' workspaces: - name: produce-something + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: produce-something workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: produce-string taskSpec: steps: - name: main args: - '----output-paths' - - $(workspaces.produce-string.path)/produce-string-Output + - $(workspaces.produce-string.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output command: - sh - -ec @@ -275,11 +285,11 @@ spec: #!/bin/sh set -exo pipefail TOTAL_SIZE=0 - ARTIFACT_SIZE=`wc -c $(workspaces.produce-string.path)/produce-string-Output | awk '{print $1}'` + ARTIFACT_SIZE=`wc -c $(workspaces.produce-string.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output | awk '{print $1}'` TOTAL_SIZE=$( expr $TOTAL_SIZE + $ARTIFACT_SIZE) touch $(results.output.path) if [[ $TOTAL_SIZE -lt 3072 ]]; then - cp $(workspaces.produce-string.path)/produce-string-Output $(results.output.path) + cp $(workspaces.produce-string.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output $(results.output.path) fi onError: continue results: @@ -310,10 +320,15 @@ spec: tekton.dev/template: '' workspaces: - name: produce-string + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: produce-string workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-anything-as-value taskSpec: steps: @@ -961,11 +976,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "anything_param" > $(workspaces.consume-anything-as-file-2.path)/anything_param + mkdir -p $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2 + echo -n "anything_param" > $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2/anything_param - name: main args: - --data - - $(workspaces.consume-anything-as-file-2.path)/anything_param + - $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2/anything_param command: - sh - -ec @@ -1005,10 +1021,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-anything-as-file-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-anything-as-file-2 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-anything-as-file-3 taskSpec: steps: @@ -1017,11 +1038,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "something_param" > $(workspaces.consume-anything-as-file-3.path)/something_param + mkdir -p $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3 + echo -n "something_param" > $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3/something_param - name: main args: - --data - - $(workspaces.consume-anything-as-file-3.path)/something_param + - $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3/something_param command: - sh - -ec @@ -1061,10 +1083,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-anything-as-file-3 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-anything-as-file-3 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-anything-as-file-4 taskSpec: steps: @@ -1073,11 +1100,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "string_param" > $(workspaces.consume-anything-as-file-4.path)/string_param + mkdir -p $(workspaces.consume-anything-as-file-4.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-4 + echo -n "string_param" > $(workspaces.consume-anything-as-file-4.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-4/string_param - name: main args: - --data - - $(workspaces.consume-anything-as-file-4.path)/string_param + - $(workspaces.consume-anything-as-file-4.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-4/string_param command: - sh - -ec @@ -1117,10 +1145,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-anything-as-file-4 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-anything-as-file-4 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-something-as-file-2 taskSpec: steps: @@ -1129,11 +1162,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "anything_param" > $(workspaces.consume-something-as-file-2.path)/anything_param + mkdir -p $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2 + echo -n "anything_param" > $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2/anything_param - name: main args: - --data - - $(workspaces.consume-something-as-file-2.path)/anything_param + - $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2/anything_param command: - sh - -ec @@ -1173,10 +1207,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-something-as-file-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-something-as-file-2 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-something-as-file-3 taskSpec: steps: @@ -1185,11 +1224,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "something_param" > $(workspaces.consume-something-as-file-3.path)/something_param + mkdir -p $(workspaces.consume-something-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-3 + echo -n "something_param" > $(workspaces.consume-something-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-3/something_param - name: main args: - --data - - $(workspaces.consume-something-as-file-3.path)/something_param + - $(workspaces.consume-something-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-3/something_param command: - sh - -ec @@ -1229,10 +1269,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-something-as-file-3 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-something-as-file-3 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-string-as-file-2 taskSpec: steps: @@ -1241,11 +1286,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "anything_param" > $(workspaces.consume-string-as-file-2.path)/anything_param + mkdir -p $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2 + echo -n "anything_param" > $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2/anything_param - name: main args: - --data - - $(workspaces.consume-string-as-file-2.path)/anything_param + - $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2/anything_param command: - sh - -ec @@ -1285,10 +1331,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-string-as-file-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-string-as-file-2 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-string-as-file-3 taskSpec: steps: @@ -1297,11 +1348,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "string_param" > $(workspaces.consume-string-as-file-3.path)/string_param + mkdir -p $(workspaces.consume-string-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-3 + echo -n "string_param" > $(workspaces.consume-string-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-3/string_param - name: main args: - --data - - $(workspaces.consume-string-as-file-3.path)/string_param + - $(workspaces.consume-string-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-3/string_param command: - sh - -ec @@ -1341,10 +1393,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-string-as-file-3 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-string-as-file-3 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-anything-as-value-5 params: - name: produce-anything-data @@ -1687,7 +1744,7 @@ spec: - name: main args: - --data - - $(workspaces.consume-anything-as-file-5.path)/produce-anything-data + - $(workspaces.consume-anything-as-file-5.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data command: - sh - -ec @@ -1727,19 +1784,24 @@ spec: tekton.dev/template: '' workspaces: - name: consume-anything-as-file-5 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-anything-as-file-5 workspace: data-passing-pipeline runAfter: - produce-anything + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-anything-as-file-6 taskSpec: steps: - name: main args: - --data - - $(workspaces.consume-anything-as-file-6.path)/produce-something-data + - $(workspaces.consume-anything-as-file-6.path)/artifacts/$(params.pipelineRun-name)/produce-something/data command: - sh - -ec @@ -1779,19 +1841,24 @@ spec: tekton.dev/template: '' workspaces: - name: consume-anything-as-file-6 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-anything-as-file-6 workspace: data-passing-pipeline runAfter: - produce-something + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-anything-as-file-7 taskSpec: steps: - name: main args: - --data - - $(workspaces.consume-anything-as-file-7.path)/produce-string-Output + - $(workspaces.consume-anything-as-file-7.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output command: - sh - -ec @@ -1831,19 +1898,24 @@ spec: tekton.dev/template: '' workspaces: - name: consume-anything-as-file-7 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-anything-as-file-7 workspace: data-passing-pipeline runAfter: - produce-string + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-something-as-file-4 taskSpec: steps: - name: main args: - --data - - $(workspaces.consume-something-as-file-4.path)/produce-anything-data + - $(workspaces.consume-something-as-file-4.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data command: - sh - -ec @@ -1883,19 +1955,24 @@ spec: tekton.dev/template: '' workspaces: - name: consume-something-as-file-4 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-something-as-file-4 workspace: data-passing-pipeline runAfter: - produce-anything + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-something-as-file-5 taskSpec: steps: - name: main args: - --data - - $(workspaces.consume-something-as-file-5.path)/produce-something-data + - $(workspaces.consume-something-as-file-5.path)/artifacts/$(params.pipelineRun-name)/produce-something/data command: - sh - -ec @@ -1935,19 +2012,24 @@ spec: tekton.dev/template: '' workspaces: - name: consume-something-as-file-5 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-something-as-file-5 workspace: data-passing-pipeline runAfter: - produce-something + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-string-as-file-4 taskSpec: steps: - name: main args: - --data - - $(workspaces.consume-string-as-file-4.path)/produce-anything-data + - $(workspaces.consume-string-as-file-4.path)/artifacts/$(params.pipelineRun-name)/produce-anything/data command: - sh - -ec @@ -1987,19 +2069,24 @@ spec: tekton.dev/template: '' workspaces: - name: consume-string-as-file-4 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-string-as-file-4 workspace: data-passing-pipeline runAfter: - produce-anything + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-string-as-file-5 taskSpec: steps: - name: main args: - --data - - $(workspaces.consume-string-as-file-5.path)/produce-string-Output + - $(workspaces.consume-string-as-file-5.path)/artifacts/$(params.pipelineRun-name)/produce-string/Output command: - sh - -ec @@ -2039,12 +2126,17 @@ spec: tekton.dev/template: '' workspaces: - name: consume-string-as-file-5 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-string-as-file-5 workspace: data-passing-pipeline runAfter: - produce-string + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) workspaces: - name: data-passing-pipeline timeout: 0s diff --git a/sdk/python/tests/compiler/testdata/data_passing_pipeline_param_as_file.yaml b/sdk/python/tests/compiler/testdata/data_passing_pipeline_param_as_file.yaml index 954a7e2f2..70731eb21 100644 --- a/sdk/python/tests/compiler/testdata/data_passing_pipeline_param_as_file.yaml +++ b/sdk/python/tests/compiler/testdata/data_passing_pipeline_param_as_file.yaml @@ -56,11 +56,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "anything_param" > $(workspaces.consume-anything-as-file.path)/anything_param + mkdir -p $(workspaces.consume-anything-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file + echo -n "anything_param" > $(workspaces.consume-anything-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file/anything_param - name: main args: - --data - - $(workspaces.consume-anything-as-file.path)/anything_param + - $(workspaces.consume-anything-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file/anything_param command: - sh - -ec @@ -100,10 +101,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-anything-as-file + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-anything-as-file workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-anything-as-file-2 taskSpec: steps: @@ -112,11 +118,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "something_param" > $(workspaces.consume-anything-as-file-2.path)/something_param + mkdir -p $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2 + echo -n "something_param" > $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2/something_param - name: main args: - --data - - $(workspaces.consume-anything-as-file-2.path)/something_param + - $(workspaces.consume-anything-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-2/something_param command: - sh - -ec @@ -156,10 +163,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-anything-as-file-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-anything-as-file-2 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-anything-as-file-3 taskSpec: steps: @@ -168,11 +180,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "string_param" > $(workspaces.consume-anything-as-file-3.path)/string_param + mkdir -p $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3 + echo -n "string_param" > $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3/string_param - name: main args: - --data - - $(workspaces.consume-anything-as-file-3.path)/string_param + - $(workspaces.consume-anything-as-file-3.path)/artifacts/$(params.pipelineRun-name)/consume-anything-as-file-3/string_param command: - sh - -ec @@ -212,10 +225,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-anything-as-file-3 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-anything-as-file-3 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-something-as-file taskSpec: steps: @@ -224,11 +242,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "anything_param" > $(workspaces.consume-something-as-file.path)/anything_param + mkdir -p $(workspaces.consume-something-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file + echo -n "anything_param" > $(workspaces.consume-something-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file/anything_param - name: main args: - --data - - $(workspaces.consume-something-as-file.path)/anything_param + - $(workspaces.consume-something-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file/anything_param command: - sh - -ec @@ -268,10 +287,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-something-as-file + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-something-as-file workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-something-as-file-2 taskSpec: steps: @@ -280,11 +304,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "something_param" > $(workspaces.consume-something-as-file-2.path)/something_param + mkdir -p $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2 + echo -n "something_param" > $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2/something_param - name: main args: - --data - - $(workspaces.consume-something-as-file-2.path)/something_param + - $(workspaces.consume-something-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-something-as-file-2/something_param command: - sh - -ec @@ -324,10 +349,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-something-as-file-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-something-as-file-2 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-string-as-file taskSpec: steps: @@ -336,11 +366,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "anything_param" > $(workspaces.consume-string-as-file.path)/anything_param + mkdir -p $(workspaces.consume-string-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file + echo -n "anything_param" > $(workspaces.consume-string-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file/anything_param - name: main args: - --data - - $(workspaces.consume-string-as-file.path)/anything_param + - $(workspaces.consume-string-as-file.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file/anything_param command: - sh - -ec @@ -380,10 +411,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-string-as-file + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-string-as-file workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) - name: consume-string-as-file-2 taskSpec: steps: @@ -392,11 +428,12 @@ spec: script: | #!/bin/sh set -exo pipefail - echo -n "string_param" > $(workspaces.consume-string-as-file-2.path)/string_param + mkdir -p $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2 + echo -n "string_param" > $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2/string_param - name: main args: - --data - - $(workspaces.consume-string-as-file-2.path)/string_param + - $(workspaces.consume-string-as-file-2.path)/artifacts/$(params.pipelineRun-name)/consume-string-as-file-2/string_param command: - sh - -ec @@ -436,10 +473,15 @@ spec: tekton.dev/template: '' workspaces: - name: consume-string-as-file-2 + params: + - name: pipelineRun-name timeout: 0s workspaces: - name: consume-string-as-file-2 workspace: data-passing-pipeline + params: + - name: pipelineRun-name + value: $(context.pipelineRun.name) workspaces: - name: data-passing-pipeline timeout: 0s