SDK - Preserving the pipeline input information in the compiled Workflow (#1381)

* SDK - Preserving the pipeline metadata in the compiled Workflow

* Stabilizing the DSL compiler tests

sdk/python/kfp/compiler/compiler.py

@@ -655,6 +655,10 @@ class Compiler(object):
     op_transformers = [add_pod_env]
     op_transformers.extend(p.conf.op_transformers)
     workflow = self._create_pipeline_workflow(args_list_with_defaults, p, op_transformers)
+
+    import json
+    workflow.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/pipeline_spec'] = json.dumps(pipeline_meta.to_dict(), sort_keys=True)
+
     return workflow
 
   def compile(self, pipeline_func, package_path, type_check=True):

sdk/python/tests/compiler/testdata/add_pod_env.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Test adding pod env",
+      "inputs": [], "name": "Test adding pod env"}'
   generateName: test-adding-pod-env-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/artifact_location.yaml

@@ -14,6 +14,11 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "hello world", "inputs":
+      [{"default": null, "description": "", "name": "tag", "type": ""}, {"default":
+      "kubeflow", "description": "", "name": "namespace", "type": ""}, {"default":
+      "foobar", "description": "", "name": "bucket", "type": ""}], "name": "foo"}'
   generateName: foo-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/basic.yaml

@@ -14,6 +14,8 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"default": null, "description": "", "name": "message", "type": ""}, {"default": null, "description": "", "name": "outputpath", "type": ""}], "name": "Save Most Frequent"}'
   generateName: save-most-frequent-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/coin.yaml

@@ -14,6 +14,8 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "inputs": [], "name": "pipeline flip coin"}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/compose.yaml

@@ -14,6 +14,8 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Download and Get Most Frequent Word and Save to GCS", "inputs": [{"default": null, "description": "", "name": "url", "type": ""}, {"default": null, "description": "", "name": "outputpath", "type": ""}], "name": "Download and Save Most Frequent"}'
   generateName: download-and-save-most-frequent-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/default_value.yaml

@@ -14,6 +14,8 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with parameter and default value.", "inputs": [{"default": "gs://ml-pipeline/shakespeare1.txt", "description": "", "name": "url", "type": ""}], "name": "Default Value"}'
   generateName: default-value-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/imagepullsecrets.yaml

@@ -1,6 +1,8 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"default": null, "description": "", "name": "message", "type": ""}], "name": "Save Most Frequent"}'
   generateName: save-most-frequent-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/param_op_transform.yaml

@@ -1,6 +1,12 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Test that parameters used
+      in Op transformation functions as pod labels would be correcly identified and
+      set as arguments in he generated yaml", "inputs": [{"default": null, "description":
+      "", "name": "param", "type": ""}], "name": "Parameters in Op transformation
+      functions"}'
   generateName: parameters-in-op-transformation-functions-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/param_substitutions.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Test the same PipelineParam
+      getting substituted in multiple places", "inputs": [], "name": "Param Substitutions"}'
   generateName: param-substitutions-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/pipelineparams.yaml

@@ -14,6 +14,11 @@
 ---
 apiVersion: argoproj.io/v1alpha1
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with multiple
+      pipeline params.", "inputs": [{"default": "latest", "description": "", "name":
+      "tag", "type": ""}, {"default": 10, "description": "", "name": "sleep_ms", "type":
+      ""}], "name": "PipelineParams"}'
   generateName: pipelineparams-
 spec:
   entrypoint: pipelineparams

sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml

@@ -1,6 +1,8 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "inputs": [], "name": "pipeline flip coin"}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/recursive_do_while.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use graph_component.",
+      "inputs": [], "name": "pipeline flip coin"}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/recursive_while.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.",
+      "inputs": [{"default": 12, "description": "", "name": "maxVal", "type": ""}],
+      "name": "pipeline flip coin"}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/resourceop_basic.yaml

@@ -1,6 +1,11 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "A Basic Example on ResourceOp
+      Usage.", "inputs": [{"default": null, "description": "", "name": "username",
+      "type": ""}, {"default": null, "description": "", "name": "password", "type":
+      ""}], "name": "ResourceOp Basic"}'
   generateName: resourceop-basic-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/retry.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use ContainerOp
+      set_retry().", "inputs": [], "name": "pipeline includes two steps which fail
+      randomly."}'
   generateName: pipeline-includes-two-steps-which-fail-randomly-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/sidecar.yaml

@@ -13,6 +13,9 @@
 # limitations under the License.
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with sidecars.",
+      "inputs": [], "name": "Sidecar"}'
   generateName: sidecar-
 apiVersion: argoproj.io/v1alpha1
 spec:

sdk/python/tests/compiler/testdata/timeout.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use ContainerOp
+      set_retry().", "inputs": [], "name": "pipeline includes two steps which fail
+      randomly."}'
   generateName: pipeline-includes-two-steps-which-fail-randomly-
 spec:
   activeDeadlineSeconds: 50

sdk/python/tests/compiler/testdata/volume.yaml

@@ -14,6 +14,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with volume.",
+      "inputs": [], "name": "Volume"}'
   generateName: volume-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "The fifth example of the
+      design doc.", "inputs": [{"default": null, "description": "", "name": "rok_url",
+      "type": ""}], "name": "VolumeSnapshotOp RokURL"}'
   generateName: volumesnapshotop-rokurl-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "The fourth example of
+      the design doc.", "inputs": [{"default": null, "description": "", "name": "url",
+      "type": ""}], "name": "VolumeSnapshotOp Sequential"}'
   generateName: volumesnapshotop-sequential-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_basic.yaml

@@ -1,6 +1,10 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "A Basic Example on VolumeOp
+      Usage.", "inputs": [{"default": null, "description": "", "name": "size", "type":
+      ""}], "name": "VolumeOp Basic"}'
   generateName: volumeop-basic-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_dag.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "The second example of
+      the design doc.", "inputs": [], "name": "Volume Op DAG"}'
   generateName: volume-op-dag-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_parallel.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "The first example of the
+      design doc.", "inputs": [], "name": "VolumeOp Parallel"}'
   generateName: volumeop-parallel-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_sequential.yaml

@@ -1,6 +1,9 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
+  annotations:
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "The third example of the
+      design doc.", "inputs": [], "name": "VolumeOp Sequential"}'
   generateName: volumeop-sequential-
 spec:
   arguments: