81 lines
2.6 KiB
Python
81 lines
2.6 KiB
Python
from pathlib import Path
|
|
|
|
import kfp.deprecated as kfp
|
|
from kfp.deprecated.components import load_component_from_file, create_component_from_func
|
|
from typing import NamedTuple
|
|
|
|
test_data_dir = Path(__file__).parent / 'test_data'
|
|
producer_op = load_component_from_file(
|
|
str(test_data_dir / 'produce_2.component.yaml'))
|
|
processor_op = load_component_from_file(
|
|
str(test_data_dir / 'process_2_2.component.yaml'))
|
|
consumer_op = load_component_from_file(
|
|
str(test_data_dir / 'consume_2.component.yaml'))
|
|
|
|
|
|
def metadata_and_metrics() -> NamedTuple(
|
|
"Outputs",
|
|
[("mlpipeline_ui_metadata", "UI_metadata"), ("mlpipeline_metrics", "Metrics"
|
|
)],
|
|
):
|
|
metadata = {
|
|
"outputs": [{
|
|
"storage": "inline",
|
|
"source": "*this should be bold*",
|
|
"type": "markdown"
|
|
}]
|
|
}
|
|
metrics = {
|
|
"metrics": [
|
|
{
|
|
"name": "train-accuracy",
|
|
"numberValue": 0.9,
|
|
},
|
|
{
|
|
"name": "test-accuracy",
|
|
"numberValue": 0.7,
|
|
},
|
|
]
|
|
}
|
|
from collections import namedtuple
|
|
import json
|
|
|
|
return namedtuple("output",
|
|
["mlpipeline_ui_metadata", "mlpipeline_metrics"])(
|
|
json.dumps(metadata), json.dumps(metrics))
|
|
|
|
|
|
@kfp.dsl.pipeline()
|
|
def artifact_passing_pipeline():
|
|
producer_task = producer_op()
|
|
processor_task = processor_op(producer_task.outputs['output_1'],
|
|
producer_task.outputs['output_2'])
|
|
consumer_task = consumer_op(processor_task.outputs['output_1'],
|
|
processor_task.outputs['output_2'])
|
|
|
|
markdown_task = create_component_from_func(func=metadata_and_metrics)()
|
|
# This line is only needed for compiling using dsl-compile to work
|
|
kfp.dsl.get_pipeline_conf(
|
|
).data_passing_method = volume_based_data_passing_method
|
|
|
|
|
|
from kubernetes.client.models import V1Volume, V1PersistentVolumeClaimVolumeSource
|
|
from kfp.deprecated.dsl import data_passing_methods
|
|
|
|
volume_based_data_passing_method = data_passing_methods.KubernetesVolume(
|
|
volume=V1Volume(
|
|
name='data',
|
|
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
|
|
claim_name='data-volume',),
|
|
),
|
|
path_prefix='artifact_data/',
|
|
)
|
|
|
|
if __name__ == '__main__':
|
|
pipeline_conf = kfp.dsl.PipelineConf()
|
|
pipeline_conf.data_passing_method = volume_based_data_passing_method
|
|
kfp.compiler.Compiler().compile(
|
|
artifact_passing_pipeline,
|
|
__file__ + '.yaml',
|
|
pipeline_conf=pipeline_conf)
|