diff --git a/pipelines-demo/dsl/data-passing-runs-graph.jpg b/pipelines-demo/dsl/data-passing-runs-graph.jpg new file mode 100644 index 00000000..3795da2c Binary files /dev/null and b/pipelines-demo/dsl/data-passing-runs-graph.jpg differ diff --git a/pipelines-demo/dsl/data-passing-runs-params.jpg b/pipelines-demo/dsl/data-passing-runs-params.jpg new file mode 100644 index 00000000..dd64b9cc Binary files /dev/null and b/pipelines-demo/dsl/data-passing-runs-params.jpg differ diff --git a/pipelines-demo/dsl/data-passing.py b/pipelines-demo/dsl/data-passing.py new file mode 100644 index 00000000..b77a4460 --- /dev/null +++ b/pipelines-demo/dsl/data-passing.py @@ -0,0 +1,57 @@ +# Copyright 2023 kbthu. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kfp +from kfp import dsl +from kfp.components import func_to_container_op +from typing import NamedTuple + + +@func_to_container_op +def calculate_op(a: float, b: float) -> NamedTuple( + 'MyOutputs', + [ + ('sum', float), + ('product', float) + ]): + return (a + b, a * b) + + +def echo_func(sum: float, product: float): + cop = dsl.ContainerOp( + name='echo_func', + image='bash:5.1', + command=['sh', '-c'], + arguments=['echo sum %s product %s | tee /out.txt' % (sum, product)], + file_outputs={'out': '/out.txt'}, + ) + cop.container.set_image_pull_policy('IfNotPresent') + return cop + + +@dsl.pipeline( + name='Kubeflow data passing example', + description='Demonstrate the data passing between Kubeflow funcs' +) +def data_passing( + a: dsl.PipelineParam(name='a', value=2), + b: dsl.PipelineParam(name='b', value=3) + ): + result = calculate_op(a, b) + echo_func(result.outputs['sum'], result.outputs['product']) + + +if __name__ == '__main__': + import kfp.compiler as compiler + compiler.Compiler().compile(data_passing, __file__ + '.yaml') diff --git a/pipelines-demo/dsl/sidecar-example.py b/pipelines-demo/dsl/sidecar-example.py new file mode 100644 index 00000000..f49d9f0b --- /dev/null +++ b/pipelines-demo/dsl/sidecar-example.py @@ -0,0 +1,32 @@ +# Copyright 2023 kbthu. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kfp +from kfp import dsl + +@dsl.pipeline( + name='metric-test', + description='query metrics from promehteus') +def exec_pipeline(): + op = dsl.ContainerOp(name='metrics-test', + image='kbthu/metrics-example:1.0.1', + command=['metrics'], + arguments=['pq', '-q', 'count(prometheus_target_interval_length_seconds)'], + ) + op.container.set_image_pull_policy('IfNotPresent') + op.add_sidecar(dsl.Sidecar('prometheus', 'bitnami/prometheus:2.44.0', command='ls') + .set_image_pull_policy('IfNotPresent')) + +if __name__ == '__main__': + kfp.compiler.Compiler().compile(exec_pipeline, __file__ + '.yaml') diff --git a/pipelines-demo/volume/runs-graph.jpg b/pipelines-demo/volume/runs-graph.jpg new file mode 100644 index 00000000..a6cc0572 Binary files /dev/null and b/pipelines-demo/volume/runs-graph.jpg differ diff --git a/pipelines-demo/volume/volume_example.py b/pipelines-demo/volume/volume_example.py new file mode 100644 index 00000000..59ac2f1a --- /dev/null +++ b/pipelines-demo/volume/volume_example.py @@ -0,0 +1,63 @@ +# Copyright 2023 kbthu. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kfp +from kfp import dsl + +def create_pv(): + return dsl.VolumeOp( + name="create_pv", + resource_name="kfp-pvc", + size="1Gi", + modes=dsl.VOLUME_MODE_RWO + ) + + +def generate_data(vol_name: str): + cop = dsl.ContainerOp( + name='generate_data', + image='bash:5.1', + command=['sh', '-c'], + arguments=['echo $(( $RANDOM % 10 + 1 )) | tee /mnt/out.txt'] + ) + cop.container.set_image_pull_policy('IfNotPresent') + cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)}) + return cop + + +def use_pre_data(vol_name: str): + cop = dsl.ContainerOp( + name='use_pre_data', + image='bash:5.1', + command=['sh', '-c'], + arguments=['tail /mnt/out.txt'] + ) + cop.container.set_image_pull_policy('IfNotPresent') + cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)}) + return cop + + +@dsl.pipeline( + name="Kubeflow volume example", + description="Demonstrate the use case of volume on Kubeflow pipeline." +) +def volume_example(): + vop = create_pv() + cop = generate_data(vop.outputs["name"]).after(vop) + use_pre_data(vop.outputs["name"]).after(vop,cop) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volume_example, __file__ + ".yaml") \ No newline at end of file