kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(sdk): Add interface for enable_caching at task level. (#6007) 

* feat(sdk): Add pipeline level caching options

* Update golden

* fixing caching options

* fix tests

* add a caching disabled case

Co-authored-by: Chen Sun <chensun@users.noreply.github.com>
This commit is contained in:
Yaqi Ji 2021-07-13 17:33:58 -07:00 committed by GitHub
parent a43af4112b
commit c6cb8acf7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 333 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
sdk/python/kfp/compiler/_default_transformers.py
View File

@ -52,6 +52,12 @@ def add_kfp_pod_env(op: BaseOp) -> BaseOp:
value_from=k8s_client.
V1EnvVarSource(field_ref=k8s_client.V1ObjectFieldSelector(
field_path="metadata.labels['workflows.argoproj.io/workflow']")))
).add_env_variable(
k8s_client.V1EnvVar(
name='ENABLE_CACHING',
value_from=k8s_client.
V1EnvVarSource(field_ref=k8s_client.V1ObjectFieldSelector(
field_path="metadata.labels['pipelines.kubeflow.org/enable_caching']")))
)
return op

4
sdk/python/kfp/compiler/_op_to_template.py
View File

@ -180,6 +180,10 @@ def _op_to_template(op: BaseOp):
if op.display_name:
op.add_pod_annotation('pipelines.kubeflow.org/task_display_name', op.display_name)
# Caching option
op.add_pod_label(
'pipelines.kubeflow.org/enable_caching', str(op.enable_caching).lower())
# NOTE in-place update to BaseOp
# replace all PipelineParams with template var strings
processed_op = _process_base_ops(op)

10
sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml vendored
View File

@ -4,7 +4,7 @@ metadata:
generateName: my-test-pipeline-
annotations:
pipelines.kubeflow.org/kfp_sdk_version: 1.6.4
pipelines.kubeflow.org/pipeline_compilation_time: '2021-07-05T13:06:29.779267'
pipelines.kubeflow.org/pipeline_compilation_time: '2021-07-13T00:27:44.352978'
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "gs://output-directory/v2-artifacts",
"name": "pipeline-output-directory"}, {"default": "my-test-pipeline", "name":
"pipeline-name"}], "name": "my-test-pipeline"}'
@ -94,6 +94,9 @@ spec:
- name: WORKFLOW_ID
valueFrom:
fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'}
- name: ENABLE_CACHING
valueFrom:
fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'}
- {name: KFP_V2_IMAGE, value: 'python:3.9'}
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"some_int": {"type":
"INT"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters":
@ -125,6 +128,7 @@ spec:
pipelines.kubeflow.org/kfp_sdk_version: 1.6.4
pipelines.kubeflow.org/pipeline-sdk-type: kfp
pipelines.kubeflow.org/v2_component: "true"
pipelines.kubeflow.org/enable_caching: "true"
initContainers:
- command: [/bin/mount_launcher.sh]
image: gcr.io/ml-pipeline/kfp-launcher:1.6.4
@ -188,6 +192,9 @@ spec:
- name: WORKFLOW_ID
valueFrom:
fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'}
- name: ENABLE_CACHING
valueFrom:
fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'}
- {name: KFP_V2_IMAGE, value: 'python:3.7'}
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type":
"INT"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data",
@ -218,6 +225,7 @@ spec:
pipelines.kubeflow.org/kfp_sdk_version: 1.6.4
pipelines.kubeflow.org/pipeline-sdk-type: kfp
pipelines.kubeflow.org/v2_component: "true"
pipelines.kubeflow.org/enable_caching: "true"
initContainers:
- command: [/bin/mount_launcher.sh]
image: gcr.io/ml-pipeline/kfp-launcher:1.6.4

10
sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml vendored
View File

@ -4,7 +4,7 @@ metadata:
generateName: my-test-pipeline-with-custom-launcher-
annotations:
pipelines.kubeflow.org/kfp_sdk_version: 1.6.4
pipelines.kubeflow.org/pipeline_compilation_time: '2021-07-05T13:06:29.620804'
pipelines.kubeflow.org/pipeline_compilation_time: '2021-07-13T00:27:44.204633'
pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "gs://output-directory/v2-artifacts",
"name": "pipeline-output-directory"}, {"default": "my-test-pipeline-with-custom-launcher",
"name": "pipeline-name"}], "name": "my-test-pipeline-with-custom-launcher"}'
@ -94,6 +94,9 @@ spec:
- name: WORKFLOW_ID
valueFrom:
fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'}
- name: ENABLE_CACHING
valueFrom:
fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'}
- {name: KFP_V2_IMAGE, value: 'python:3.9'}
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"some_int": {"type":
"INT"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters":
@ -125,6 +128,7 @@ spec:
pipelines.kubeflow.org/kfp_sdk_version: 1.6.4
pipelines.kubeflow.org/pipeline-sdk-type: kfp
pipelines.kubeflow.org/v2_component: "true"
pipelines.kubeflow.org/enable_caching: "true"
initContainers:
- command: [/bin/mount_launcher.sh]
image: my-custom-image
@ -188,6 +192,9 @@ spec:
- name: WORKFLOW_ID
valueFrom:
fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'}
- name: ENABLE_CACHING
valueFrom:
fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'}
- {name: KFP_V2_IMAGE, value: 'python:3.7'}
- {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type":
"INT"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data",
@ -218,6 +225,7 @@ spec:
pipelines.kubeflow.org/kfp_sdk_version: 1.6.4
pipelines.kubeflow.org/pipeline-sdk-type: kfp
pipelines.kubeflow.org/v2_component: "true"
pipelines.kubeflow.org/enable_caching: "true"
initContainers:
- command: [/bin/mount_launcher.sh]
image: my-custom-image

21
sdk/python/kfp/dsl/_container_op.py
View File

@ -297,7 +297,7 @@ class Container(V1Container):
"E", "P", "T", "G", "M", "K".
"""
if not isinstance(memory,_pipeline_param.PipelineParam):
if not isinstance(memory,_pipeline_param.PipelineParam):
self._validate_size_string(memory)
return self.add_resource_request('memory', memory)
@ -341,7 +341,7 @@ class Container(V1Container):
cpu(Union[str, PipelineParam]): A string which can be a number or a number followed by "m", which
means 1/1000.
"""
if not isinstance(cpu,_pipeline_param.PipelineParam):
if not isinstance(cpu,_pipeline_param.PipelineParam):
self._validate_cpu_string(cpu)
return self.add_resource_request('cpu', cpu)
@ -353,7 +353,7 @@ class Container(V1Container):
means 1/1000.
"""
if not isinstance(cpu,_pipeline_param.PipelineParam):
if not isinstance(cpu,_pipeline_param.PipelineParam):
self._validate_cpu_string(cpu)
if self._container_spec:
self._container_spec.resources.cpu_limit = _get_cpu_number(cpu)
@ -810,6 +810,9 @@ class BaseOp(object):
self._inputs = []
self.dependent_names = []
# Caching option, default to True
self.enable_caching = True
@property
def inputs(self):
"""List of PipelineParams that will be converted into input parameters
@ -1003,6 +1006,18 @@ class BaseOp(object):
self.display_name = name
return self
def set_caching_options(self, enable_caching: bool) -> 'BaseOp':
"""Sets caching options for the Op.
Args:
enable_caching: Whether or not to enable caching for this task.
Returns:
Self return to allow chained setting calls.
"""
self.enable_caching = enable_caching
return self
def __repr__(self):
return str({self.__class__.__name__: self.__dict__})

3
sdk/python/kfp/v2/compiler/compiler.py
View File

@ -659,6 +659,9 @@ class Compiler(object):
deployment_config.executors[importer_exec_label].importer.CopyFrom(
subgroup.importer_spec)
# Task level caching option.
subgroup.task_spec.caching_options.enable_cache = subgroup.enable_caching
subgroup_inputs = inputs.get(subgroup.name, [])
subgroup_params = [param for param, _ in subgroup_inputs]

8
sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json
View File

@ -158,6 +158,9 @@
"dag": {
"tasks": {
"preprocess": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-preprocess"
},
@ -173,6 +176,9 @@
}
},
"train": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-train"
},
@ -243,7 +249,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.3"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

14
sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json
View File

@ -212,6 +212,9 @@
},
"tasks": {
"add-numbers": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-add-numbers"
},
@ -230,6 +233,9 @@
}
},
"concat-message": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-concat-message"
},
@ -248,6 +254,9 @@
}
},
"output-artifact": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-output-artifact"
},
@ -276,6 +285,9 @@
}
},
"output-named-tuple": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-output-named-tuple"
},
@ -325,7 +337,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.3"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

11
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json
View File

@ -76,6 +76,9 @@
"dag": {
"tasks": {
"print-text": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-text"
},
@ -95,6 +98,9 @@
}
},
"print-text-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-text-2"
},
@ -117,6 +123,9 @@
}
},
"print-text-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-text-3"
},
@ -143,7 +152,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

5
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json
View File

@ -32,6 +32,9 @@
"dag": {
"tasks": {
"component-with-concat-placeholder": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-concat-placeholder"
},
@ -54,7 +57,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

13
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json
View File

@ -5,6 +5,7 @@
"dag": {
"tasks": {
"flip-coin-2": {
"cachingOptions": {},
"componentRef": {
"name": "comp-flip-coin-2"
},
@ -13,6 +14,9 @@
}
},
"print-msg-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-msg-2"
},
@ -34,6 +38,9 @@
}
},
"print-msg-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-msg-3"
},
@ -225,6 +232,7 @@
}
},
"flip-coin": {
"cachingOptions": {},
"componentRef": {
"name": "comp-flip-coin"
},
@ -233,6 +241,9 @@
}
},
"print-msg": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-msg"
},
@ -264,7 +275,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

4
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.py
View File

@ -36,11 +36,11 @@ print_op = components.create_component_from_func(print_msg)
@dsl.pipeline(name='single-condition-pipeline', pipeline_root='dummy_root')
def my_pipeline(text: str = 'condition test'):
flip1 = flip_coin_op()
flip1 = flip_coin_op().set_caching_options(False)
print_op(flip1.output)
with dsl.Condition(flip1.output == 'heads'):
flip2 = flip_coin_op()
flip2 = flip_coin_op().set_caching_options(False)
print_op(flip2.output)
print_op(text)

8
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_custom_job_spec.json
View File

@ -116,6 +116,9 @@
"dag": {
"tasks": {
"training-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-training-op"
},
@ -135,6 +138,9 @@
}
},
"training-op-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-training-op-2"
},
@ -157,7 +163,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.3"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

8
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json
View File

@ -5,6 +5,9 @@
"dag": {
"tasks": {
"fail-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-fail-op"
},
@ -24,6 +27,9 @@
}
},
"print-op-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-2"
},
@ -197,7 +203,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.3"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

5
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json
View File

@ -39,6 +39,9 @@
"dag": {
"tasks": {
"component-with-optional-inputs": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-optional-inputs"
},
@ -73,7 +76,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

14
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json
View File

@ -5,6 +5,9 @@
"dag": {
"tasks": {
"importer-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-importer-2"
},
@ -20,6 +23,9 @@
}
},
"train-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-train-2"
},
@ -250,6 +256,9 @@
}
},
"importer": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-importer"
},
@ -269,6 +278,9 @@
}
},
"train": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-train"
},
@ -300,7 +312,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.3"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

14
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_output.json
View File

@ -15,6 +15,9 @@
"dag": {
"tasks": {
"print-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op"
},
@ -30,6 +33,9 @@
}
},
"print-op-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-2"
},
@ -46,6 +52,9 @@
}
},
"print-op-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-3"
},
@ -176,6 +185,9 @@
"dag": {
"tasks": {
"args-generator-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-args-generator-op"
},
@ -214,7 +226,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

11
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_parameter.json
View File

@ -5,6 +5,9 @@
"dag": {
"tasks": {
"print-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op"
},
@ -20,6 +23,9 @@
}
},
"print-op-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-2"
},
@ -36,6 +42,9 @@
}
},
"print-op-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-3"
},
@ -182,7 +191,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

11
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loop_static.json
View File

@ -5,6 +5,9 @@
"dag": {
"tasks": {
"print-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op"
},
@ -20,6 +23,9 @@
}
},
"print-op-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-2"
},
@ -36,6 +42,9 @@
}
},
"print-op-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-3"
},
@ -165,7 +174,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

26
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json
View File

@ -15,6 +15,9 @@
"dag": {
"tasks": {
"args-generator-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-args-generator-op"
},
@ -72,6 +75,9 @@
"dag": {
"tasks": {
"print-op-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-2"
},
@ -103,6 +109,9 @@
"dag": {
"tasks": {
"print-op-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-3"
},
@ -171,6 +180,9 @@
"dag": {
"tasks": {
"print-op-5": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-5"
},
@ -186,6 +198,9 @@
}
},
"print-op-6": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-6"
},
@ -293,6 +308,9 @@
}
},
"print-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op"
},
@ -330,6 +348,9 @@
"dag": {
"tasks": {
"print-op-4": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-4"
},
@ -615,6 +636,9 @@
}
},
"flip-coin-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-flip-coin-op"
},
@ -654,7 +678,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

8
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json
View File

@ -17,6 +17,9 @@
},
"tasks": {
"output-metrics-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-output-metrics-2"
},
@ -151,6 +154,9 @@
}
},
"output-metrics": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-output-metrics"
},
@ -176,7 +182,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.3"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

26
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json
View File

@ -35,6 +35,9 @@
}
},
"flip-coin-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-flip-coin-3"
},
@ -43,6 +46,9 @@
}
},
"print-msg-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-msg-3"
},
@ -80,6 +86,9 @@
"dag": {
"tasks": {
"flip-coin-4": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-flip-coin-4"
},
@ -88,6 +97,9 @@
}
},
"print-msg-4": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-msg-4"
},
@ -367,6 +379,9 @@
}
},
"flip-coin": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-flip-coin"
},
@ -375,6 +390,9 @@
}
},
"flip-coin-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-flip-coin-2"
},
@ -383,6 +401,9 @@
}
},
"print-msg": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-msg"
},
@ -404,6 +425,9 @@
}
},
"print-msg-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-msg-2"
},
@ -428,7 +452,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

23
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json
View File

@ -59,6 +59,9 @@
}
},
"generate-random-number": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-generate-random-number"
},
@ -80,6 +83,9 @@
"dag": {
"tasks": {
"print": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print"
},
@ -118,6 +124,9 @@
"dag": {
"tasks": {
"print-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-2"
},
@ -210,6 +219,9 @@
}
},
"generate-random-number-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-generate-random-number-2"
},
@ -231,6 +243,9 @@
"dag": {
"tasks": {
"print-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-3"
},
@ -269,6 +284,9 @@
"dag": {
"tasks": {
"print-4": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-4"
},
@ -512,6 +530,9 @@
}
},
"flip-coin": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-flip-coin"
},
@ -523,7 +544,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

5
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json
View File

@ -43,6 +43,9 @@
"dag": {
"tasks": {
"print-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op"
},
@ -139,7 +142,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

8
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json
View File

@ -87,6 +87,9 @@
"dag": {
"tasks": {
"ingestion": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-ingestion"
},
@ -102,6 +105,9 @@
}
},
"trainer": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-trainer"
},
@ -147,7 +153,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

11
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json
View File

@ -5,6 +5,9 @@
"dag": {
"tasks": {
"print-op2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op2"
},
@ -182,6 +185,9 @@
}
},
"print-op": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op"
},
@ -204,6 +210,9 @@
}
},
"print-op-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-print-op-2"
},
@ -242,7 +251,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.2"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

8
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json
View File

@ -95,6 +95,9 @@
"dag": {
"tasks": {
"ingestion": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-ingestion"
},
@ -110,6 +113,9 @@
}
},
"trainer": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-trainer"
},
@ -155,7 +161,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

11
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json
View File

@ -112,6 +112,9 @@
"dag": {
"tasks": {
"add": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-add"
},
@ -134,6 +137,9 @@
}
},
"add-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-add-2"
},
@ -158,6 +164,9 @@
}
},
"add-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-add-3"
},
@ -199,7 +208,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

8
sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json
View File

@ -145,6 +145,9 @@
"dag": {
"tasks": {
"downstream": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-downstream"
},
@ -204,6 +207,9 @@
}
},
"upstream": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-upstream"
},
@ -248,7 +254,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

8
sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json
View File

@ -67,6 +67,9 @@
"dag": {
"tasks": {
"read-from-gcs": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-read-from-gcs"
},
@ -88,6 +91,9 @@
}
},
"write-to-gcs": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-write-to-gcs"
},
@ -113,7 +119,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.0-rc.0"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root",

26
sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json
View File

@ -487,6 +487,9 @@
"dag": {
"tasks": {
"chicago-taxi-trips-dataset": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-chicago-taxi-trips-dataset"
},
@ -527,6 +530,9 @@
}
},
"convert-csv-to-apache-parquet": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-convert-csv-to-apache-parquet"
},
@ -548,6 +554,9 @@
}
},
"xgboost-predict": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-xgboost-predict"
},
@ -585,6 +594,9 @@
}
},
"xgboost-predict-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-xgboost-predict-2"
},
@ -622,6 +634,9 @@
}
},
"xgboost-predict-3": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-xgboost-predict-3"
},
@ -659,6 +674,9 @@
}
},
"xgboost-predict-4": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-xgboost-predict-4"
},
@ -696,6 +714,9 @@
}
},
"xgboost-train": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-xgboost-train"
},
@ -768,6 +789,9 @@
}
},
"xgboost-train-2": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-xgboost-train-2"
},
@ -843,7 +867,7 @@
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "kfp-1.6.3"
"sdkVersion": "kfp-1.6.4"
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"

12
sdk/python/tests/compiler/compiler_tests.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Optional
import unittest
import kfp
import kfp.compiler as compiler
@ -113,6 +114,9 @@ class TestCompiler(parameterized.TestCase):
{'name': 'echo-merged',
'valueFrom': {'path': '/tmp/message.txt'}
}],
},
'metadata': {
'labels': {'pipelines.kubeflow.org/enable_caching': 'true'}
}
}
res_output = {
@ -151,6 +155,9 @@ class TestCompiler(parameterized.TestCase):
" name: resource\n"
),
'setOwnerReference': True
},
'metadata': {
'labels': {'pipelines.kubeflow.org/enable_caching': 'true'}
}
}
@ -699,6 +706,8 @@ class TestCompiler(parameterized.TestCase):
del compiled_template['name'], expected['name']
for output in compiled_template['outputs'].get('parameters', []) + compiled_template['outputs'].get('artifacts', []) + expected['outputs'].get('parameters', []) + expected['outputs'].get('artifacts', []):
del output['name']
del compiled_template['metadata']
assert compiled_template == expected
def test_tolerations(self):
@ -1231,3 +1240,6 @@ implementation:
# compare
self.assertEqual(pipeline_yaml_arg, pipeline_yaml_kwarg)
if __name__ == '__main__':
unittest.main()

4
sdk/python/tests/compiler/testdata/add_pod_env.yaml vendored
View File

@ -29,6 +29,10 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.labels['workflows.argoproj.io/workflow']
- name: ENABLE_CACHING
valueFrom:
fieldRef:
fieldPath: metadata.labels['pipelines.kubeflow.org/enable_caching']
image: library/bash
metadata:
labels: