fix(sdk): Fix param prefix replacement (#952)

sdk/python/kfp_tekton/compiler/compiler.py

@@ -318,7 +318,7 @@ class TektonCompiler(Compiler):
                 replace_str = param[1] + '-'
                 self.loops_pipeline[group_name]['spec']['params'].append({
                   'name': param[0], 'value': '$(tasks.%s.results.%s)' % (
-                    param[1], sanitize_k8s_name(param[0].replace(replace_str, ''))
+                    param[1], sanitize_k8s_name(param[0].replace(replace_str, '', 1))
                   )
                 })
               if not param[1]:
@@ -362,7 +362,7 @@ class TektonCompiler(Compiler):
             replace_str = param[1] + '-'
             custom_task['spec']['params'].append({
               'name': param[0], 'value': '$(tasks.%s.results.%s)' % (
-                param[1], sanitize_k8s_name(param[0].replace(replace_str, ''))
+                param[1], sanitize_k8s_name(param[0].replace(replace_str, '', 1))
               )
             })
           if not param[1] and param[0] not in param_list:
@@ -1167,7 +1167,7 @@ class TektonCompiler(Compiler):
     opgroup_name_to_parent_groups = self._get_groups_for_opsgroups(pipeline.groups[0])
     for loop_task_key in self.loops_pipeline.keys():
       task_name_prefix = '-'.join(self._group_names[:-1] + [""])
-      raw_task_key = loop_task_key.replace(task_name_prefix, "")
+      raw_task_key = loop_task_key.replace(task_name_prefix, "", 1)
       for key in opgroup_name_to_parent_groups.keys():
         if raw_task_key in key:
           raw_task_key = key

sdk/python/tests/compiler/compiler_tests.py

@@ -233,6 +233,13 @@ class TestTektonCompiler(unittest.TestCase):
     from .testdata.nested_recur_custom_task import double_recursion_test
     self._test_pipeline_workflow(double_recursion_test, 'nested_recur_custom_task.yaml')
 
+  def test_param_same_prefix_workflow(self):
+    """
+    Test compiling a param that has same task prefix workflow.
+    """
+    from .testdata.param_same_prefix import prefixes
+    self._test_pipeline_workflow(prefixes, 'param_same_prefix.yaml')
+
   def test_nested_recur_params_workflow(self):
     """
     Test compiling a nested recursive workflow.

sdk/python/tests/compiler/testdata/param_same_prefix.py

@@ -0,0 +1,170 @@
+# Copyright 2022 kubeflow.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+from typing import Mapping
+import yaml
+
+from kfp import dsl, components
+from kfp_tekton.tekton import TEKTON_CUSTOM_TASK_IMAGES, Loop
+from kfp_tekton.compiler import TektonCompiler
+
+
+ARTIFACT_FETCHER_IMAGE_NAME = "fetcher/image:latest"
+TEKTON_CUSTOM_TASK_IMAGES = TEKTON_CUSTOM_TASK_IMAGES.append(ARTIFACT_FETCHER_IMAGE_NAME)
+
+
+class Coder:
+  def empty(self):
+    return ""
+
+
+TektonCompiler._get_unique_id_code = Coder.empty
+
+
+def fetcher_op(name: str, artifact_paths: Mapping[str, str]):
+  template_yaml = {
+    'name': name,
+    'description': 'Fetch',
+    'inputs': [
+      {'name': key, 'type': 'String'}
+      for key in artifact_paths.keys()
+    ],
+    'outputs': [
+      {'name': key, 'type': 'String'}
+      for key in artifact_paths.keys()
+    ],
+    'implementation': {
+      'container': {
+        'image': ARTIFACT_FETCHER_IMAGE_NAME,
+        'command': ['sh', '-c'],  # irrelevant
+        'args': [
+          '--apiVersion', 'fetcher.tekton.dev/v1alpha1',
+          '--kind', 'FETCHER',
+          '--name', 'fetcher_op',
+          *itertools.chain(*[
+            (f'--{key}', {'inputValue': key})
+            for key in artifact_paths.keys()
+          ])
+        ]
+      }
+    }
+  }
+  template_str = yaml.dump(template_yaml, Dumper=yaml.SafeDumper)
+  template = components.load_component_from_text(template_str)
+  args = {
+    key.replace('-', '_'): val
+    for key, val in artifact_paths.items()
+  }
+  op = template(**args)
+  op.add_pod_annotation("valid_container", "false")
+  return op
+
+
+def print_op(name: str, messages: Mapping[str, str]):
+  inputs = '\n'.join([
+    '  - {name: %s, type: String}' % key for key in messages.keys()
+  ])
+  outputs = '\n'.join([
+    '  - {name: %s, type: String}' % key for key in messages.keys()
+  ])
+  inout_refs = '\n'.join(list(itertools.chain(*[
+    (
+      '      - {inputValue: %s}' % key,
+      '      - {outputPath: %s}' % key,
+    ) for key in messages.keys()
+  ])))
+  print_template_str = """
+  name: %s
+  inputs:
+%s
+  outputs:
+%s
+  implementation:
+    container:
+      image: alpine:3.6
+      command:
+      - sh
+      - -c
+      - ...
+%s
+  """ % (name, inputs, outputs, inout_refs)
+  print_template = components.load_component_from_text(
+    print_template_str
+  )
+  args = {
+    key.replace('-', '_'): val
+    for key, val in messages.items()
+  }
+  return print_template(**args)
+
+
+def string_consumer(name: str, msg: str = None):
+  if msg is None:
+    msg = name
+  template = components.load_component_from_text(
+  """
+  name: %s
+  inputs:
+  - {name: input_text, type: String}
+  outputs:
+  - {name: output_value, type: String}
+  implementation:
+    container:
+      image: alpine:3.6
+      command:
+      - sh
+      - -c
+      - |
+        set -e
+        echo $0 > $1
+      - {inputValue: input_text}
+      - {outputPath: output_value}
+  """ % name
+  )
+  return template(msg)
+
+
+@dsl.pipeline("prefixes")
+def prefixes(foo: str, li: list = [1, 2, 3]):
+
+  # custom-task, prefix diff from name
+  fetch_00 = fetcher_op('foo-00', {'bar-00-val': foo})
+
+  # custom-task, prefix same as name
+  fetch_01 = fetcher_op('foo-01', {'foo-01-val': foo})
+
+  # custom-task, param name identical to name
+  fetch_02 = fetcher_op('foo-02', {'foo-02': foo})
+
+  # normal task, prefix diff from name
+  print_10 = print_op('foo-10', {'bar-10-val': foo})
+
+  # normal task, prefix same as name
+  print_11 = print_op('foo-11', {'foo-11-val': foo})
+
+  # normal task, param name identical to name
+  print_12 = print_op('foo-12', {'foo-12': foo})
+
+  with Loop(li):
+    string_consumer('fetch-00', fetch_00.output)
+    string_consumer('fetch-01', fetch_01.output)
+    string_consumer('fetch-02', fetch_02.output)
+    string_consumer('print-10', print_10.output)
+    string_consumer('print-11', print_11.output)
+    string_consumer('print-12', print_12.output)
+
+
+if __name__ == '__main__':
+  TektonCompiler().compile(prefixes, __file__.replace('.py', '.yaml'))

sdk/python/tests/compiler/testdata/param_same_prefix.yaml

@@ -0,0 +1,430 @@
+# Copyright 2021 kubeflow.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: prefixes
+  annotations:
+    tekton.dev/output_artifacts: '{"fetch-00": [{"key": "artifacts/$PIPELINERUN/fetch-00/output_value.tgz",
+      "name": "fetch-00-output_value", "path": "/tmp/outputs/output_value/data"}],
+      "fetch-01": [{"key": "artifacts/$PIPELINERUN/fetch-01/output_value.tgz", "name":
+      "fetch-01-output_value", "path": "/tmp/outputs/output_value/data"}], "fetch-02":
+      [{"key": "artifacts/$PIPELINERUN/fetch-02/output_value.tgz", "name": "fetch-02-output_value",
+      "path": "/tmp/outputs/output_value/data"}], "foo-10": [{"key": "artifacts/$PIPELINERUN/foo-10/bar-10-val.tgz",
+      "name": "foo-10-bar-10-val", "path": "/tmp/outputs/bar-10-val/data"}], "foo-11":
+      [{"key": "artifacts/$PIPELINERUN/foo-11/val.tgz", "name": "foo-11-foo-11-val",
+      "path": "/tmp/outputs/foo-11-val/data"}], "foo-12": [{"key": "artifacts/$PIPELINERUN/foo-12/foo-12.tgz",
+      "name": "foo-12-foo-12", "path": "/tmp/outputs/foo-12/data"}], "print-10": [{"key":
+      "artifacts/$PIPELINERUN/print-10/output_value.tgz", "name": "print-10-output_value",
+      "path": "/tmp/outputs/output_value/data"}], "print-11": [{"key": "artifacts/$PIPELINERUN/print-11/output_value.tgz",
+      "name": "print-11-output_value", "path": "/tmp/outputs/output_value/data"}],
+      "print-12": [{"key": "artifacts/$PIPELINERUN/print-12/output_value.tgz", "name":
+      "print-12-output_value", "path": "/tmp/outputs/output_value/data"}]}'
+    tekton.dev/input_artifacts: '{"fetch-00": [{"name": "foo-00-bar-00-val", "parent_task":
+      "foo-00"}], "fetch-01": [{"name": "foo-01-foo-01-val", "parent_task": "foo-01"}],
+      "fetch-02": [{"name": "foo-02-foo-02", "parent_task": "foo-02"}], "print-10":
+      [{"name": "foo-10-bar-10-val", "parent_task": "foo-10"}], "print-11": [{"name":
+      "foo-11-foo-11-val", "parent_task": "foo-11"}], "print-12": [{"name": "foo-12-foo-12",
+      "parent_task": "foo-12"}]}'
+    tekton.dev/artifact_bucket: mlpipeline
+    tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
+    tekton.dev/artifact_endpoint_scheme: http://
+    tekton.dev/artifact_items: '{"fetch-00": [["output_value", "$(results.output-value.path)"]],
+      "fetch-01": [["output_value", "$(results.output-value.path)"]], "fetch-02":
+      [["output_value", "$(results.output-value.path)"]], "foo-10": [["bar-10-val",
+      "$(results.bar-10-val.path)"]], "foo-11": [["foo-11-val", "$(results.foo-11-val.path)"]],
+      "foo-12": [["foo-12", "$(results.foo-12.path)"]], "print-10": [["output_value",
+      "$(results.output-value.path)"]], "print-11": [["output_value", "$(results.output-value.path)"]],
+      "print-12": [["output_value", "$(results.output-value.path)"]]}'
+    sidecar.istio.io/inject: "false"
+    pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
+    pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"name": "foo", "type": "String"},
+      {"default": "[1, 2, 3]", "name": "li", "optional": true, "type": "JsonArray"}],
+      "name": "prefixes"}'
+spec:
+  params:
+  - name: foo
+    value: ''
+  - name: li
+    value: '[1, 2, 3]'
+  pipelineSpec:
+    params:
+    - name: foo
+    - name: li
+      default: '[1, 2, 3]'
+    tasks:
+    - name: foo-00
+      params:
+      - name: bar-00-val
+        value: $(params.foo)
+      taskRef:
+        name: fetcher_op
+        apiVersion: fetcher.tekton.dev/v1alpha1
+        kind: FETCHER
+      timeout: 525600m
+    - name: foo-01
+      params:
+      - name: foo-01-val
+        value: $(params.foo)
+      taskRef:
+        name: fetcher_op
+        apiVersion: fetcher.tekton.dev/v1alpha1
+        kind: FETCHER
+      timeout: 525600m
+    - name: foo-02
+      params:
+      - name: foo-02
+        value: $(params.foo)
+      taskRef:
+        name: fetcher_op
+        apiVersion: fetcher.tekton.dev/v1alpha1
+        kind: FETCHER
+      timeout: 525600m
+    - name: foo-10
+      params:
+      - name: foo
+        value: $(params.foo)
+      taskSpec:
+        steps:
+        - name: main
+          command:
+          - sh
+          - -c
+          - '...'
+          - $(inputs.params.foo)
+          - $(results.bar-10-val.path)
+          image: alpine:3.6
+        params:
+        - name: foo
+        results:
+        - name: bar-10-val
+          description: /tmp/outputs/bar-10-val/data
+        metadata:
+          labels:
+            pipelines.kubeflow.org/pipelinename: ''
+            pipelines.kubeflow.org/generation: ''
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "foo-10", "outputs":
+              [{"name": "bar-10-val", "type": "String"}], "version": "foo-10@sha256=d6d62cb6c6b77d1f0d609f5f5f2e0827d16f80de51fdfa5bde09e631fb759995"}'
+            tekton.dev/template: ''
+      timeout: 525600m
+    - name: foo-11
+      params:
+      - name: foo
+        value: $(params.foo)
+      taskSpec:
+        steps:
+        - name: main
+          command:
+          - sh
+          - -c
+          - '...'
+          - $(inputs.params.foo)
+          - $(results.foo-11-val.path)
+          image: alpine:3.6
+        params:
+        - name: foo
+        results:
+        - name: foo-11-val
+          description: /tmp/outputs/foo-11-val/data
+        metadata:
+          labels:
+            pipelines.kubeflow.org/pipelinename: ''
+            pipelines.kubeflow.org/generation: ''
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "foo-11", "outputs":
+              [{"name": "foo-11-val", "type": "String"}], "version": "foo-11@sha256=377008435f70950b0bc568837051eb5d49375b55d1105eb7bc9c4f7863c336a3"}'
+            tekton.dev/template: ''
+      timeout: 525600m
+    - name: foo-12
+      params:
+      - name: foo
+        value: $(params.foo)
+      taskSpec:
+        steps:
+        - name: main
+          command:
+          - sh
+          - -c
+          - '...'
+          - $(inputs.params.foo)
+          - $(results.foo-12.path)
+          image: alpine:3.6
+        params:
+        - name: foo
+        results:
+        - name: foo-12
+          description: /tmp/outputs/foo-12/data
+        metadata:
+          labels:
+            pipelines.kubeflow.org/pipelinename: ''
+            pipelines.kubeflow.org/generation: ''
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "foo-12", "outputs":
+              [{"name": "foo-12", "type": "String"}], "version": "foo-12@sha256=ce03e07a304a0a53f19878261cbde059912689fd90a0a68b61aa0821e6201480"}'
+            tekton.dev/template: ''
+      timeout: 525600m
+    - runAfter:
+      - foo-00
+      - foo-01
+      - foo-02
+      - foo-10
+      - foo-11
+      - foo-12
+      name: prefixes-for-loop-1
+      params:
+      - name: foo-00-bar-00-val
+        value: $(tasks.foo-00.results.bar-00-val)
+      - name: foo-01-foo-01-val
+        value: $(tasks.foo-01.results.foo-01-val)
+      - name: foo-02-foo-02
+        value: $(tasks.foo-02.results.foo-02)
+      - name: foo-10-bar-10-val
+        value: $(tasks.foo-10.results.bar-10-val)
+      - name: foo-11-foo-11-val
+        value: $(tasks.foo-11.results.foo-11-val)
+      - name: foo-12-foo-12
+        value: $(tasks.foo-12.results.foo-12)
+      - name: li-loop-item
+        value: $(params.li)
+      taskSpec:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: PipelineLoop
+        spec:
+          pipelineSpec:
+            params:
+            - name: foo-00-bar-00-val
+              type: string
+            - name: foo-01-foo-01-val
+              type: string
+            - name: foo-02-foo-02
+              type: string
+            - name: foo-10-bar-10-val
+              type: string
+            - name: foo-11-foo-11-val
+              type: string
+            - name: foo-12-foo-12
+              type: string
+            - name: li-loop-item
+              type: string
+            tasks:
+            - name: fetch-00
+              params:
+              - name: foo-00-bar-00-val
+                value: $(params.foo-00-bar-00-val)
+              taskSpec:
+                steps:
+                - name: main
+                  command:
+                  - sh
+                  - -c
+                  - |
+                    set -e
+                    echo $0 > $1
+                  - $(inputs.params.foo-00-bar-00-val)
+                  - $(results.output-value.path)
+                  image: alpine:3.6
+                params:
+                - name: foo-00-bar-00-val
+                  type: string
+                results:
+                - name: output-value
+                  description: /tmp/outputs/output_value/data
+                metadata:
+                  labels:
+                    pipelines.kubeflow.org/pipelinename: ''
+                    pipelines.kubeflow.org/generation: ''
+                    pipelines.kubeflow.org/cache_enabled: "true"
+                  annotations:
+                    pipelines.kubeflow.org/component_spec_digest: '{"name": "fetch-00",
+                      "outputs": [{"name": "output_value", "type": "String"}], "version":
+                      "fetch-00@sha256=7e9b72ed3ff0fe01739d5c1882d0398713f844d5b40d856da78cd272aa29224e"}'
+                    tekton.dev/template: ''
+              timeout: 525600m
+            - name: fetch-01
+              params:
+              - name: foo-01-foo-01-val
+                value: $(params.foo-01-foo-01-val)
+              taskSpec:
+                steps:
+                - name: main
+                  command:
+                  - sh
+                  - -c
+                  - |
+                    set -e
+                    echo $0 > $1
+                  - $(inputs.params.foo-01-foo-01-val)
+                  - $(results.output-value.path)
+                  image: alpine:3.6
+                params:
+                - name: foo-01-foo-01-val
+                  type: string
+                results:
+                - name: output-value
+                  description: /tmp/outputs/output_value/data
+                metadata:
+                  labels:
+                    pipelines.kubeflow.org/pipelinename: ''
+                    pipelines.kubeflow.org/generation: ''
+                    pipelines.kubeflow.org/cache_enabled: "true"
+                  annotations:
+                    pipelines.kubeflow.org/component_spec_digest: '{"name": "fetch-01",
+                      "outputs": [{"name": "output_value", "type": "String"}], "version":
+                      "fetch-01@sha256=aabda0027c4968f5bd77ca09283a4d6893a7cc0f24aba3c8ed3e736a189664c2"}'
+                    tekton.dev/template: ''
+              timeout: 525600m
+            - name: fetch-02
+              params:
+              - name: foo-02-foo-02
+                value: $(params.foo-02-foo-02)
+              taskSpec:
+                steps:
+                - name: main
+                  command:
+                  - sh
+                  - -c
+                  - |
+                    set -e
+                    echo $0 > $1
+                  - $(inputs.params.foo-02-foo-02)
+                  - $(results.output-value.path)
+                  image: alpine:3.6
+                params:
+                - name: foo-02-foo-02
+                  type: string
+                results:
+                - name: output-value
+                  description: /tmp/outputs/output_value/data
+                metadata:
+                  labels:
+                    pipelines.kubeflow.org/pipelinename: ''
+                    pipelines.kubeflow.org/generation: ''
+                    pipelines.kubeflow.org/cache_enabled: "true"
+                  annotations:
+                    pipelines.kubeflow.org/component_spec_digest: '{"name": "fetch-02",
+                      "outputs": [{"name": "output_value", "type": "String"}], "version":
+                      "fetch-02@sha256=50fd399aa3779cb4c5b268102fb26755c28079d262e8b96e13e1fc204d69b921"}'
+                    tekton.dev/template: ''
+              timeout: 525600m
+            - name: print-10
+              params:
+              - name: foo-10-bar-10-val
+                value: $(params.foo-10-bar-10-val)
+              taskSpec:
+                steps:
+                - name: main
+                  command:
+                  - sh
+                  - -c
+                  - |
+                    set -e
+                    echo $0 > $1
+                  - $(inputs.params.foo-10-bar-10-val)
+                  - $(results.output-value.path)
+                  image: alpine:3.6
+                params:
+                - name: foo-10-bar-10-val
+                  type: string
+                results:
+                - name: output-value
+                  description: /tmp/outputs/output_value/data
+                metadata:
+                  labels:
+                    pipelines.kubeflow.org/pipelinename: ''
+                    pipelines.kubeflow.org/generation: ''
+                    pipelines.kubeflow.org/cache_enabled: "true"
+                  annotations:
+                    pipelines.kubeflow.org/component_spec_digest: '{"name": "print-10",
+                      "outputs": [{"name": "output_value", "type": "String"}], "version":
+                      "print-10@sha256=d9ea1c895144dfa19d7e457987bdf79548809f7b45af8ae4560998c99e193751"}'
+                    tekton.dev/template: ''
+              timeout: 525600m
+            - name: print-11
+              params:
+              - name: foo-11-foo-11-val
+                value: $(params.foo-11-foo-11-val)
+              taskSpec:
+                steps:
+                - name: main
+                  command:
+                  - sh
+                  - -c
+                  - |
+                    set -e
+                    echo $0 > $1
+                  - $(inputs.params.foo-11-foo-11-val)
+                  - $(results.output-value.path)
+                  image: alpine:3.6
+                params:
+                - name: foo-11-foo-11-val
+                  type: string
+                results:
+                - name: output-value
+                  description: /tmp/outputs/output_value/data
+                metadata:
+                  labels:
+                    pipelines.kubeflow.org/pipelinename: ''
+                    pipelines.kubeflow.org/generation: ''
+                    pipelines.kubeflow.org/cache_enabled: "true"
+                  annotations:
+                    pipelines.kubeflow.org/component_spec_digest: '{"name": "print-11",
+                      "outputs": [{"name": "output_value", "type": "String"}], "version":
+                      "print-11@sha256=5a1013995ecd3285d03ae6b24d872d3d1ca1eaf7c78d099eca050a3ce9e3fd83"}'
+                    tekton.dev/template: ''
+              timeout: 525600m
+            - name: print-12
+              params:
+              - name: foo-12-foo-12
+                value: $(params.foo-12-foo-12)
+              taskSpec:
+                steps:
+                - name: main
+                  command:
+                  - sh
+                  - -c
+                  - |
+                    set -e
+                    echo $0 > $1
+                  - $(inputs.params.foo-12-foo-12)
+                  - $(results.output-value.path)
+                  image: alpine:3.6
+                params:
+                - name: foo-12-foo-12
+                  type: string
+                results:
+                - name: output-value
+                  description: /tmp/outputs/output_value/data
+                metadata:
+                  labels:
+                    pipelines.kubeflow.org/pipelinename: ''
+                    pipelines.kubeflow.org/generation: ''
+                    pipelines.kubeflow.org/cache_enabled: "true"
+                  annotations:
+                    pipelines.kubeflow.org/component_spec_digest: '{"name": "print-12",
+                      "outputs": [{"name": "output_value", "type": "String"}], "version":
+                      "print-12@sha256=8c34161acc91f4c5e394f75aeffd0a793db16e196aa8a246c0c26bd2336c54e6"}'
+                    tekton.dev/template: ''
+              timeout: 525600m
+          iterateParam: li-loop-item
+        metadata:
+          labels:
+            pipelines.kubeflow.org/pipelinename: ''
+            pipelines.kubeflow.org/generation: ''
+            pipelines.kubeflow.org/cache_enabled: "true"
+  timeout: 525600m

sdk/python/tests/compiler/testdata/param_same_prefix_noninlined.yaml

@@ -0,0 +1,276 @@
+# Copyright 2021 kubeflow.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: prefixes
+  annotations:
+    tekton.dev/output_artifacts: '{"fetch-00": [{"key": "artifacts/$PIPELINERUN/fetch-00/output_value.tgz",
+      "name": "fetch-00-output_value", "path": "/tmp/outputs/output_value/data"}],
+      "fetch-01": [{"key": "artifacts/$PIPELINERUN/fetch-01/output_value.tgz", "name":
+      "fetch-01-output_value", "path": "/tmp/outputs/output_value/data"}], "fetch-02":
+      [{"key": "artifacts/$PIPELINERUN/fetch-02/output_value.tgz", "name": "fetch-02-output_value",
+      "path": "/tmp/outputs/output_value/data"}], "foo-10": [{"key": "artifacts/$PIPELINERUN/foo-10/bar-10-val.tgz",
+      "name": "foo-10-bar-10-val", "path": "/tmp/outputs/bar-10-val/data"}], "foo-11":
+      [{"key": "artifacts/$PIPELINERUN/foo-11/val.tgz", "name": "foo-11-foo-11-val",
+      "path": "/tmp/outputs/foo-11-val/data"}], "foo-12": [{"key": "artifacts/$PIPELINERUN/foo-12/foo-12.tgz",
+      "name": "foo-12-foo-12", "path": "/tmp/outputs/foo-12/data"}], "print-10": [{"key":
+      "artifacts/$PIPELINERUN/print-10/output_value.tgz", "name": "print-10-output_value",
+      "path": "/tmp/outputs/output_value/data"}], "print-11": [{"key": "artifacts/$PIPELINERUN/print-11/output_value.tgz",
+      "name": "print-11-output_value", "path": "/tmp/outputs/output_value/data"}],
+      "print-12": [{"key": "artifacts/$PIPELINERUN/print-12/output_value.tgz", "name":
+      "print-12-output_value", "path": "/tmp/outputs/output_value/data"}]}'
+    tekton.dev/input_artifacts: '{"fetch-00": [{"name": "foo-00-bar-00-val", "parent_task":
+      "foo-00"}], "fetch-01": [{"name": "foo-01-foo-01-val", "parent_task": "foo-01"}],
+      "fetch-02": [{"name": "foo-02-foo-02", "parent_task": "foo-02"}], "print-10":
+      [{"name": "foo-10-bar-10-val", "parent_task": "foo-10"}], "print-11": [{"name":
+      "foo-11-foo-11-val", "parent_task": "foo-11"}], "print-12": [{"name": "foo-12-foo-12",
+      "parent_task": "foo-12"}]}'
+    tekton.dev/artifact_bucket: mlpipeline
+    tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
+    tekton.dev/artifact_endpoint_scheme: http://
+    tekton.dev/artifact_items: '{"fetch-00": [["output_value", "$(results.output-value.path)"]],
+      "fetch-01": [["output_value", "$(results.output-value.path)"]], "fetch-02":
+      [["output_value", "$(results.output-value.path)"]], "foo-10": [["bar-10-val",
+      "$(results.bar-10-val.path)"]], "foo-11": [["foo-11-val", "$(results.foo-11-val.path)"]],
+      "foo-12": [["foo-12", "$(results.foo-12.path)"]], "print-10": [["output_value",
+      "$(results.output-value.path)"]], "print-11": [["output_value", "$(results.output-value.path)"]],
+      "print-12": [["output_value", "$(results.output-value.path)"]]}'
+    sidecar.istio.io/inject: "false"
+    pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
+    pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"name": "foo", "type": "String"},
+      {"default": "[1, 2, 3]", "name": "li", "optional": true, "type": "JsonArray"}],
+      "name": "prefixes"}'
+    tekton.dev/resource_templates: '[{"apiVersion": "custom.tekton.dev/v1alpha1",
+      "kind": "PipelineLoop", "metadata": {"name": "prefixes-for-loop-1"}, "spec":
+      {"iterateParam": "li-loop-item", "pipelineSpec": {"params": [{"name": "foo-00-bar-00-val",
+      "type": "string"}, {"name": "foo-01-foo-01-val", "type": "string"}, {"name":
+      "foo-02-foo-02", "type": "string"}, {"name": "foo-10-bar-10-val", "type": "string"},
+      {"name": "foo-11-foo-11-val", "type": "string"}, {"name": "foo-12-foo-12", "type":
+      "string"}, {"name": "li-loop-item", "type": "string"}], "tasks": [{"name": "fetch-00",
+      "params": [{"name": "foo-00-bar-00-val", "value": "$(params.foo-00-bar-00-val)"}],
+      "taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
+      "{\"name\": \"fetch-00\", \"outputs\": [{\"name\": \"output_value\", \"type\":
+      \"String\"}], \"version\": \"fetch-00@sha256=7e9b72ed3ff0fe01739d5c1882d0398713f844d5b40d856da78cd272aa29224e\"}",
+      "tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
+      "true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
+      ""}}, "params": [{"name": "foo-00-bar-00-val", "type": "string"}], "results":
+      [{"description": "/tmp/outputs/output_value/data", "name": "output-value"}],
+      "steps": [{"command": ["sh", "-c", "set -e\necho $0 > $1\n", "$(inputs.params.foo-00-bar-00-val)",
+      "$(results.output-value.path)"], "image": "alpine:3.6", "name": "main"}]}, "timeout":
+      "525600m"}, {"name": "fetch-01", "params": [{"name": "foo-01-foo-01-val", "value":
+      "$(params.foo-01-foo-01-val)"}], "taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
+      "{\"name\": \"fetch-01\", \"outputs\": [{\"name\": \"output_value\", \"type\":
+      \"String\"}], \"version\": \"fetch-01@sha256=aabda0027c4968f5bd77ca09283a4d6893a7cc0f24aba3c8ed3e736a189664c2\"}",
+      "tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
+      "true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
+      ""}}, "params": [{"name": "foo-01-foo-01-val", "type": "string"}], "results":
+      [{"description": "/tmp/outputs/output_value/data", "name": "output-value"}],
+      "steps": [{"command": ["sh", "-c", "set -e\necho $0 > $1\n", "$(inputs.params.foo-01-foo-01-val)",
+      "$(results.output-value.path)"], "image": "alpine:3.6", "name": "main"}]}, "timeout":
+      "525600m"}, {"name": "fetch-02", "params": [{"name": "foo-02-foo-02", "value":
+      "$(params.foo-02-foo-02)"}], "taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
+      "{\"name\": \"fetch-02\", \"outputs\": [{\"name\": \"output_value\", \"type\":
+      \"String\"}], \"version\": \"fetch-02@sha256=50fd399aa3779cb4c5b268102fb26755c28079d262e8b96e13e1fc204d69b921\"}",
+      "tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
+      "true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
+      ""}}, "params": [{"name": "foo-02-foo-02", "type": "string"}], "results": [{"description":
+      "/tmp/outputs/output_value/data", "name": "output-value"}], "steps": [{"command":
+      ["sh", "-c", "set -e\necho $0 > $1\n", "$(inputs.params.foo-02-foo-02)", "$(results.output-value.path)"],
+      "image": "alpine:3.6", "name": "main"}]}, "timeout": "525600m"}, {"name": "print-10",
+      "params": [{"name": "foo-10-bar-10-val", "value": "$(params.foo-10-bar-10-val)"}],
+      "taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
+      "{\"name\": \"print-10\", \"outputs\": [{\"name\": \"output_value\", \"type\":
+      \"String\"}], \"version\": \"print-10@sha256=d9ea1c895144dfa19d7e457987bdf79548809f7b45af8ae4560998c99e193751\"}",
+      "tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
+      "true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
+      ""}}, "params": [{"name": "foo-10-bar-10-val", "type": "string"}], "results":
+      [{"description": "/tmp/outputs/output_value/data", "name": "output-value"}],
+      "steps": [{"command": ["sh", "-c", "set -e\necho $0 > $1\n", "$(inputs.params.foo-10-bar-10-val)",
+      "$(results.output-value.path)"], "image": "alpine:3.6", "name": "main"}]}, "timeout":
+      "525600m"}, {"name": "print-11", "params": [{"name": "foo-11-foo-11-val", "value":
+      "$(params.foo-11-foo-11-val)"}], "taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
+      "{\"name\": \"print-11\", \"outputs\": [{\"name\": \"output_value\", \"type\":
+      \"String\"}], \"version\": \"print-11@sha256=5a1013995ecd3285d03ae6b24d872d3d1ca1eaf7c78d099eca050a3ce9e3fd83\"}",
+      "tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
+      "true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
+      ""}}, "params": [{"name": "foo-11-foo-11-val", "type": "string"}], "results":
+      [{"description": "/tmp/outputs/output_value/data", "name": "output-value"}],
+      "steps": [{"command": ["sh", "-c", "set -e\necho $0 > $1\n", "$(inputs.params.foo-11-foo-11-val)",
+      "$(results.output-value.path)"], "image": "alpine:3.6", "name": "main"}]}, "timeout":
+      "525600m"}, {"name": "print-12", "params": [{"name": "foo-12-foo-12", "value":
+      "$(params.foo-12-foo-12)"}], "taskSpec": {"metadata": {"annotations": {"pipelines.kubeflow.org/component_spec_digest":
+      "{\"name\": \"print-12\", \"outputs\": [{\"name\": \"output_value\", \"type\":
+      \"String\"}], \"version\": \"print-12@sha256=8c34161acc91f4c5e394f75aeffd0a793db16e196aa8a246c0c26bd2336c54e6\"}",
+      "tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
+      "true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
+      ""}}, "params": [{"name": "foo-12-foo-12", "type": "string"}], "results": [{"description":
+      "/tmp/outputs/output_value/data", "name": "output-value"}], "steps": [{"command":
+      ["sh", "-c", "set -e\necho $0 > $1\n", "$(inputs.params.foo-12-foo-12)", "$(results.output-value.path)"],
+      "image": "alpine:3.6", "name": "main"}]}, "timeout": "525600m"}]}}}]'
+spec:
+  params:
+  - name: foo
+    value: ''
+  - name: li
+    value: '[1, 2, 3]'
+  pipelineSpec:
+    params:
+    - name: foo
+    - name: li
+      default: '[1, 2, 3]'
+    tasks:
+    - name: foo-00
+      params:
+      - name: bar-00-val
+        value: $(params.foo)
+      taskRef:
+        name: fetcher_op
+        apiVersion: fetcher.tekton.dev/v1alpha1
+        kind: FETCHER
+      timeout: 525600m
+    - name: foo-01
+      params:
+      - name: foo-01-val
+        value: $(params.foo)
+      taskRef:
+        name: fetcher_op
+        apiVersion: fetcher.tekton.dev/v1alpha1
+        kind: FETCHER
+      timeout: 525600m
+    - name: foo-02
+      params:
+      - name: foo-02
+        value: $(params.foo)
+      taskRef:
+        name: fetcher_op
+        apiVersion: fetcher.tekton.dev/v1alpha1
+        kind: FETCHER
+      timeout: 525600m
+    - name: foo-10
+      params:
+      - name: foo
+        value: $(params.foo)
+      taskSpec:
+        steps:
+        - name: main
+          command:
+          - sh
+          - -c
+          - '...'
+          - $(inputs.params.foo)
+          - $(results.bar-10-val.path)
+          image: alpine:3.6
+        params:
+        - name: foo
+        results:
+        - name: bar-10-val
+          description: /tmp/outputs/bar-10-val/data
+        metadata:
+          labels:
+            pipelines.kubeflow.org/pipelinename: ''
+            pipelines.kubeflow.org/generation: ''
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "foo-10", "outputs":
+              [{"name": "bar-10-val", "type": "String"}], "version": "foo-10@sha256=d6d62cb6c6b77d1f0d609f5f5f2e0827d16f80de51fdfa5bde09e631fb759995"}'
+            tekton.dev/template: ''
+      timeout: 525600m
+    - name: foo-11
+      params:
+      - name: foo
+        value: $(params.foo)
+      taskSpec:
+        steps:
+        - name: main
+          command:
+          - sh
+          - -c
+          - '...'
+          - $(inputs.params.foo)
+          - $(results.foo-11-val.path)
+          image: alpine:3.6
+        params:
+        - name: foo
+        results:
+        - name: foo-11-val
+          description: /tmp/outputs/foo-11-val/data
+        metadata:
+          labels:
+            pipelines.kubeflow.org/pipelinename: ''
+            pipelines.kubeflow.org/generation: ''
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "foo-11", "outputs":
+              [{"name": "foo-11-val", "type": "String"}], "version": "foo-11@sha256=377008435f70950b0bc568837051eb5d49375b55d1105eb7bc9c4f7863c336a3"}'
+            tekton.dev/template: ''
+      timeout: 525600m
+    - name: foo-12
+      params:
+      - name: foo
+        value: $(params.foo)
+      taskSpec:
+        steps:
+        - name: main
+          command:
+          - sh
+          - -c
+          - '...'
+          - $(inputs.params.foo)
+          - $(results.foo-12.path)
+          image: alpine:3.6
+        params:
+        - name: foo
+        results:
+        - name: foo-12
+          description: /tmp/outputs/foo-12/data
+        metadata:
+          labels:
+            pipelines.kubeflow.org/pipelinename: ''
+            pipelines.kubeflow.org/generation: ''
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "foo-12", "outputs":
+              [{"name": "foo-12", "type": "String"}], "version": "foo-12@sha256=ce03e07a304a0a53f19878261cbde059912689fd90a0a68b61aa0821e6201480"}'
+            tekton.dev/template: ''
+      timeout: 525600m
+    - runAfter:
+      - foo-00
+      - foo-01
+      - foo-02
+      - foo-10
+      - foo-11
+      - foo-12
+      name: prefixes-for-loop-1
+      taskRef:
+        apiVersion: custom.tekton.dev/v1alpha1
+        kind: PipelineLoop
+        name: prefixes-for-loop-1
+      params:
+      - name: foo-00-bar-00-val
+        value: $(tasks.foo-00.results.bar-00-val)
+      - name: foo-01-foo-01-val
+        value: $(tasks.foo-01.results.foo-01-val)
+      - name: foo-02-foo-02
+        value: $(tasks.foo-02.results.foo-02)
+      - name: foo-10-bar-10-val
+        value: $(tasks.foo-10.results.bar-10-val)
+      - name: foo-11-foo-11-val
+        value: $(tasks.foo-11.results.foo-11-val)
+      - name: foo-12-foo-12
+        value: $(tasks.foo-12.results.foo-12)
+      - name: li-loop-item
+        value: $(params.li)
+  timeout: 525600m