121 lines
3.4 KiB
YAML
121 lines
3.4 KiB
YAML
# Copyright 2018 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: Workflow
|
|
metadata:
|
|
annotations:
|
|
pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"name": "message", "type": "String"}, {"name": "outputpath", "type": "String"}], "name": "Save Most Frequent"}'
|
|
generateName: save-most-frequent-
|
|
spec:
|
|
arguments:
|
|
parameters:
|
|
- name: message
|
|
- name: outputpath
|
|
entrypoint: save-most-frequent
|
|
serviceAccountName: pipeline-runner
|
|
onExit: exiting
|
|
templates:
|
|
- dag:
|
|
tasks:
|
|
- arguments:
|
|
parameters:
|
|
- name: message
|
|
value: '{{inputs.parameters.message}}'
|
|
name: get-frequent
|
|
template: get-frequent
|
|
- arguments:
|
|
parameters:
|
|
- name: get-frequent-word
|
|
value: '{{tasks.get-frequent.outputs.parameters.get-frequent-word}}'
|
|
- name: outputpath
|
|
value: '{{inputs.parameters.outputpath}}'
|
|
dependencies:
|
|
- get-frequent
|
|
name: save
|
|
template: save
|
|
inputs:
|
|
parameters:
|
|
- name: message
|
|
- name: outputpath
|
|
name: exit-handler-1
|
|
- container:
|
|
args:
|
|
- echo exit!
|
|
command:
|
|
- sh
|
|
- -c
|
|
image: python:3.5-jessie
|
|
name: exiting
|
|
- container:
|
|
args:
|
|
- python -c "from collections import Counter; words = Counter('{{inputs.parameters.message}}'.split());
|
|
print(max(words, key=words.get))" | tee /tmp/message.txt
|
|
command:
|
|
- sh
|
|
- -c
|
|
image: python:3.5-jessie
|
|
resources:
|
|
requests:
|
|
memory: 200M
|
|
inputs:
|
|
parameters:
|
|
- name: message
|
|
name: get-frequent
|
|
outputs:
|
|
artifacts:
|
|
- name: get-frequent-word
|
|
path: /tmp/message.txt
|
|
parameters:
|
|
- name: get-frequent-word
|
|
valueFrom:
|
|
path: /tmp/message.txt
|
|
- container:
|
|
args:
|
|
- echo {{inputs.parameters.get-frequent-word}} | tee /tmp/results.txt | gsutil
|
|
cp /tmp/results.txt {{inputs.parameters.outputpath}}
|
|
command:
|
|
- sh
|
|
- -c
|
|
image: google/cloud-sdk
|
|
resources:
|
|
limits:
|
|
cloud-tpus.google.com/v2: "8"
|
|
cpu: "0.5"
|
|
nvidia.com/gpu: "2"
|
|
inputs:
|
|
parameters:
|
|
- name: get-frequent-word
|
|
- name: outputpath
|
|
metadata:
|
|
annotations:
|
|
tf-version.cloud-tpus.google.com: "1.12"
|
|
name: save
|
|
nodeSelector:
|
|
cloud.google.com/gke-accelerator: nvidia-tesla-k80
|
|
- dag:
|
|
tasks:
|
|
- arguments:
|
|
parameters:
|
|
- name: message
|
|
value: '{{inputs.parameters.message}}'
|
|
- name: outputpath
|
|
value: '{{inputs.parameters.outputpath}}'
|
|
name: exit-handler-1
|
|
template: exit-handler-1
|
|
inputs:
|
|
parameters:
|
|
- name: message
|
|
- name: outputpath
|
|
name: save-most-frequent
|