From fe0bc678a833d7d732fc5894b3c12a7f71f7175a Mon Sep 17 00:00:00 2001 From: Ning Date: Fri, 6 Sep 2019 11:46:15 -0700 Subject: [PATCH] add more core samples in the sample test (#2030) * add more core samples in the sample test * remove retry test for now * update pipeline transformer sample * add default value to volume_ops and fix error * remove volume snapshot and resource op sample tests, which require platform support * remove volume_ops since it is already included in the e2e * add timeout --- .../pipeline_transformers.py | 22 +++++++++---------- .../{resourceop_basic.py => resource_ops.py} | 0 samples/core/sidecar/sidecar.py | 4 ++-- ..._snapshot_op.py => volume_snapshot_ops.py} | 0 test/sample_test.yaml | 3 +++ 5 files changed, 15 insertions(+), 14 deletions(-) rename samples/core/resource_ops/{resourceop_basic.py => resource_ops.py} (100%) rename samples/core/volume_snapshot_ops/{volume_snapshot_op.py => volume_snapshot_ops.py} (100%) diff --git a/samples/core/pipeline_transformers/pipeline_transformers.py b/samples/core/pipeline_transformers/pipeline_transformers.py index 73a7aadddb..ca1481affe 100644 --- a/samples/core/pipeline_transformers/pipeline_transformers.py +++ b/samples/core/pipeline_transformers/pipeline_transformers.py @@ -17,18 +17,16 @@ import kfp from kfp import dsl - -def random_failure_op(exit_codes): - """A component that fails randomly.""" +def print_op(msg): + """Print a message.""" return dsl.ContainerOp( - name='random_failure', - image='python:alpine3.6', - command=['python', '-c'], - arguments=['import random; import sys; exit_code = random.choice(sys.argv[1].split(",")); print(exit_code); sys.exit(exit_code)', exit_codes] + name='Print', + image='alpine:3.6', + command=['echo', msg], ) -def add_retry(op): - op.set_retry(5) +def add_annotation(op): + op.add_pod_annotation(name='hobby', value='football') return op @dsl.pipeline( @@ -36,9 +34,9 @@ def add_retry(op): description='The pipeline includes two steps which fail randomly. It shows how to use ContainerOp(...).set_retry(...).' ) def retry_sample_pipeline(): - op1 = random_failure_op('0,1,2,3') - op2 = random_failure_op('0,1') - dsl.get_pipeline_conf().add_op_transformer(add_retry) + op1 = print_op('hey, what are you up to?') + op2 = print_op('train my model.') + dsl.get_pipeline_conf().add_op_transformer(add_annotation) if __name__ == '__main__': kfp.compiler.Compiler().compile(retry_sample_pipeline, __file__ + '.zip') diff --git a/samples/core/resource_ops/resourceop_basic.py b/samples/core/resource_ops/resource_ops.py similarity index 100% rename from samples/core/resource_ops/resourceop_basic.py rename to samples/core/resource_ops/resource_ops.py diff --git a/samples/core/sidecar/sidecar.py b/samples/core/sidecar/sidecar.py index 096ec47503..35665d2c11 100644 --- a/samples/core/sidecar/sidecar.py +++ b/samples/core/sidecar/sidecar.py @@ -20,7 +20,7 @@ import kfp.dsl as dsl name="pipeline_with_sidecar", description="A pipeline that demonstrates how to add a sidecar to an operation." ) -def pipeline_with_sidecar(sleep_ms: int = 10): +def pipeline_with_sidecar(sleep_sec: int = 30): # sidecar with sevice that reply "hello world" to any GET request echo = dsl.Sidecar( @@ -35,7 +35,7 @@ def pipeline_with_sidecar(sleep_ms: int = 10): image="busybox:latest", command=["sh", "-c"], arguments=[ - "sleep %s; wget localhost:5678 -O /tmp/results.txt" % sleep_ms + "sleep %s; wget localhost:5678 -O /tmp/results.txt" % sleep_sec ], # sleep for X sec and call the sidecar and save results to output sidecars=[echo], file_outputs={"downloaded": "/tmp/results.txt"}, diff --git a/samples/core/volume_snapshot_ops/volume_snapshot_op.py b/samples/core/volume_snapshot_ops/volume_snapshot_ops.py similarity index 100% rename from samples/core/volume_snapshot_ops/volume_snapshot_op.py rename to samples/core/volume_snapshot_ops/volume_snapshot_ops.py diff --git a/test/sample_test.yaml b/test/sample_test.yaml index c3af360094..f1949bb7ba 100644 --- a/test/sample_test.yaml +++ b/test/sample_test.yaml @@ -71,6 +71,9 @@ spec: - xgboost_training_cm - lightweight_component - dsl_static_type_checking + - pipeline_transformers + - secret + - sidecar # Build and push image - name: build-image-by-dockerfile inputs: