kubeflow
/
examples
mirror of https://github.com/kubeflow/examples.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'kubeflow:master' into master

This commit is contained in:
kbthu 2023-05-30 14:35:27 +08:00 committed by GitHub
parent 130db716df 0ae74ee99d
commit ccf17dcc05
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 505 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
cnn template/Pipeline template mnist example.docx Normal file
View File

Binary file not shown.

159
cnn template/cnn temp.py Normal file
View File

@ -0,0 +1,159 @@
#!/usr/bin/env python
# coding: utf-8
# In[2]:
with open("requirements.txt", "w") as f:
f.write("kfp==1.8.9\n")
get_ipython().system('pip install -r requirements.txt --upgrade --user')
# In[1]:
from typing import NamedTuple
import kfp
from kfp import dsl
from kfp.components import func_to_container_op, InputPath, OutputPath
from typing import NamedTuple
def train(log_folder:str) -> NamedTuple('Outputs', [('logdir', str)]):
print('mnist_func:', log_folder)
import tensorflow as tf
import json
mnist = tf.keras.datasets.mnist
(x_train,y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train/255.0, x_test/255.0
def create_model():
return tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape = (28,28)),
tf.keras.layers.Dense(512, activation = 'relu'),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation = 'softmax')
])
model = create_model()
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
import datetime
import os
### add tensorboard logout callback
log_dir = os.path.join(log_folder, "logs", datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
######
model.fit(x=x_train,
y=y_train,
epochs=5,
validation_data=(x_test, y_test),
callbacks=[tensorboard_callback])
print('At least tensorboard callbacks are correct')
print('logdir:', log_dir)
return ([log_dir])
def prepare_tensorboard_from_localdir(pvc_name:str) -> NamedTuple('Outputs', [('mlpipeline_ui_metadata', 'UI_metadata')]):
import json
import kfp.components as components
prepare_tensorboard = components.load_component_from_url(
'https://raw.githubusercontent.com/kubeflow/pipelines/1b107eb4bb2510ecb99fd5f4fb438cbf7c96a87a/components/contrib/tensorflow/tensorboard/prepare_tensorboard/component.yaml'
)
# log_dir_uri is consisted of volume:///folders
# the volume.name should be same as ones specified in pod_template.spec.volumes.name
return prepare_tensorboard(
log_dir_uri=f'volume://mypvc/logs',
image="footprintai/tensorboard:2.7.0",
pod_template_spec=json.dumps({
"spec": {
"containers": [
{
"volumeMounts": [
{
"mountPath": "/data",
"name": "mypvc"
}
]
}
],
"serviceAccountName": "default-editor",
"volumes": [
{
"name": "mypvc",
"persistentVolumeClaim": {
"claimName": pvc_name
}
}
]
}
}),
)
def model(text1):
return dsl.ContainerOp(
name='model',
image='library/bash:4.4.23',
command=['sh', '-c'],
arguments=['echo "$0"', text1])
# In[2]:
import kfp.dsl as dsl
import kfp.components as components
import time
@dsl.pipeline(
name='tf pipeline',
description='A pipeline to train a model on tf dataset and start a tensorboard.'
)
def tf_pipeline(text1='message 1'):
log_folder = '/data'
pvc_name = 'input'
unique_pvc_resource_name = 'my-awesome-kf-workshop-%d'% int(time.time())
vop = dsl.VolumeOp(
name=pvc_name,
resource_name=unique_pvc_resource_name,
size="1Gi",
modes=dsl.VOLUME_MODE_RWO,
generate_unique_name=False,
)
tf_op = func_to_container_op(
func=train,
base_image="tensorflow/tensorflow:2.0.0-py3",
)
tensorboard_task = prepare_tensorboard_from_localdir(unique_pvc_resource_name)
tf_task = tf_op(log_folder).add_pvolumes({
log_folder:vop.volume,
})
step1_task = model(text1)
# step1_task.after(tensorboard_task)
tensorboard_task.after(tf_task)
tf_task.after(step1_task)
# In[4]:
kfp.compiler.Compiler().compile(tf_pipeline, 'helloworld.zip')
# In[ ]:

BIN
cnn template/cnn template.docx Normal file
View File

Binary file not shown.

346
cnn template/cnntemp.ipynb Normal file
View File

@ -0,0 +1,346 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"id": "w3cO7lQMtLkh",
"outputId": "267a2cff-6127-4946-8f3d-2ef927df4f29"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
"Collecting kfp==1.8.9\n",
" Downloading kfp-1.8.9.tar.gz (296 kB)\n",
"\u001b[K |████████████████████████████████| 296 kB 20.1 MB/s \n",
"\u001b[?25hCollecting absl-py<=0.11,>=0.9\n",
" Downloading absl_py-0.11.0-py3-none-any.whl (127 kB)\n",
"\u001b[K |████████████████████████████████| 127 kB 73.9 MB/s \n",
"\u001b[?25hCollecting PyYAML<6,>=5.3\n",
" Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
"\u001b[K |████████████████████████████████| 636 kB 57.3 MB/s \n",
"\u001b[?25hCollecting google-cloud-storage<2,>=1.20.0\n",
" Downloading google_cloud_storage-1.44.0-py2.py3-none-any.whl (106 kB)\n",
"\u001b[K |████████████████████████████████| 106 kB 64.8 MB/s \n",
"\u001b[?25hCollecting kubernetes<19,>=8.0.0\n",
" Downloading kubernetes-18.20.0-py2.py3-none-any.whl (1.6 MB)\n",
"\u001b[K |████████████████████████████████| 1.6 MB 64.0 MB/s \n",
"\u001b[?25hRequirement already satisfied: google-api-python-client<2,>=1.7.8 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (1.12.11)\n",
"Collecting google-auth<2,>=1.6.1\n",
" Downloading google_auth-1.35.0-py2.py3-none-any.whl (152 kB)\n",
"\u001b[K |████████████████████████████████| 152 kB 66.3 MB/s \n",
"\u001b[?25hCollecting requests-toolbelt<1,>=0.8.0\n",
" Downloading requests_toolbelt-0.10.1-py2.py3-none-any.whl (54 kB)\n",
"\u001b[K |████████████████████████████████| 54 kB 3.4 MB/s \n",
"\u001b[?25hCollecting cloudpickle<3,>=2.0.0\n",
" Downloading cloudpickle-2.2.0-py3-none-any.whl (25 kB)\n",
"Collecting kfp-server-api<2.0.0,>=1.1.2\n",
" Downloading kfp-server-api-1.8.5.tar.gz (58 kB)\n",
"\u001b[K |████████████████████████████████| 58 kB 6.1 MB/s \n",
"\u001b[?25hCollecting jsonschema<4,>=3.0.1\n",
" Downloading jsonschema-3.2.0-py2.py3-none-any.whl (56 kB)\n",
"\u001b[K |████████████████████████████████| 56 kB 5.3 MB/s \n",
"\u001b[?25hRequirement already satisfied: tabulate<1,>=0.8.6 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (0.8.10)\n",
"Requirement already satisfied: click<9,>=7.1.2 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (7.1.2)\n",
"Collecting Deprecated<2,>=1.2.7\n",
" Downloading Deprecated-1.2.13-py2.py3-none-any.whl (9.6 kB)\n",
"Collecting strip-hints<1,>=0.1.8\n",
" Downloading strip-hints-0.1.10.tar.gz (29 kB)\n",
"Collecting docstring-parser<1,>=0.7.3\n",
" Downloading docstring_parser-0.15-py3-none-any.whl (36 kB)\n",
"Collecting kfp-pipeline-spec<0.2.0,>=0.1.13\n",
" Downloading kfp_pipeline_spec-0.1.16-py3-none-any.whl (19 kB)\n",
"Collecting fire<1,>=0.3.1\n",
" Downloading fire-0.4.0.tar.gz (87 kB)\n",
"\u001b[K |████████████████████████████████| 87 kB 7.8 MB/s \n",
"\u001b[?25hRequirement already satisfied: protobuf<4,>=3.13.0 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (3.19.6)\n",
"Requirement already satisfied: uritemplate<4,>=3.0.1 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (3.0.1)\n",
"Requirement already satisfied: pydantic<2,>=1.8.2 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (1.10.2)\n",
"Requirement already satisfied: typer<1.0,>=0.3.2 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (0.4.2)\n",
"Collecting typing-extensions<4,>=3.7.4\n",
" Downloading typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)\n",
"Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py<=0.11,>=0.9->kfp==1.8.9->-r requirements.txt (line 1)) (1.15.0)\n",
"Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.7/dist-packages (from Deprecated<2,>=1.2.7->kfp==1.8.9->-r requirements.txt (line 1)) (1.14.1)\n",
"Requirement already satisfied: termcolor in /usr/local/lib/python3.7/dist-packages (from fire<1,>=0.3.1->kfp==1.8.9->-r requirements.txt (line 1)) (2.1.0)\n",
"Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (0.0.4)\n",
"Requirement already satisfied: google-api-core<3dev,>=1.21.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (2.8.2)\n",
"Requirement already satisfied: httplib2<1dev,>=0.15.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (0.17.4)\n",
"Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /usr/local/lib/python3.7/dist-packages (from google-api-core<3dev,>=1.21.0->google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (1.56.4)\n",
"Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<3dev,>=1.21.0->google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (2.23.0)\n",
"Collecting cachetools<5.0,>=2.0.0\n",
" Downloading cachetools-4.2.4-py3-none-any.whl (10 kB)\n",
"Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.1->kfp==1.8.9->-r requirements.txt (line 1)) (4.9)\n",
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.1->kfp==1.8.9->-r requirements.txt (line 1)) (0.2.8)\n",
"Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.1->kfp==1.8.9->-r requirements.txt (line 1)) (57.4.0)\n",
"Requirement already satisfied: google-cloud-core<3.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.9->-r requirements.txt (line 1)) (2.3.2)\n",
"Requirement already satisfied: google-resumable-media<3.0dev,>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.9->-r requirements.txt (line 1)) (2.4.0)\n",
"Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.7/dist-packages (from google-resumable-media<3.0dev,>=1.3.0->google-cloud-storage<2,>=1.20.0->kfp==1.8.9->-r requirements.txt (line 1)) (1.5.0)\n",
"Requirement already satisfied: pyrsistent>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema<4,>=3.0.1->kfp==1.8.9->-r requirements.txt (line 1)) (0.19.2)\n",
"Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from jsonschema<4,>=3.0.1->kfp==1.8.9->-r requirements.txt (line 1)) (4.13.0)\n",
"Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema<4,>=3.0.1->kfp==1.8.9->-r requirements.txt (line 1)) (22.1.0)\n",
"Requirement already satisfied: urllib3>=1.15 in /usr/local/lib/python3.7/dist-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.9->-r requirements.txt (line 1)) (1.24.3)\n",
"Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.9->-r requirements.txt (line 1)) (2022.9.24)\n",
"Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.9->-r requirements.txt (line 1)) (2.8.2)\n",
"Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.7/dist-packages (from kubernetes<19,>=8.0.0->kfp==1.8.9->-r requirements.txt (line 1)) (1.3.1)\n",
"Collecting websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0\n",
" Downloading websocket_client-1.4.2-py3-none-any.whl (55 kB)\n",
"\u001b[K |████████████████████████████████| 55 kB 4.1 MB/s \n",
"\u001b[?25hRequirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.1->kfp==1.8.9->-r requirements.txt (line 1)) (0.4.8)\n",
"Collecting pydantic<2,>=1.8.2\n",
" Downloading pydantic-1.10.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.8 MB)\n",
"\u001b[K |████████████████████████████████| 11.8 MB 34.0 MB/s \n",
"\u001b[?25h Downloading pydantic-1.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.8 MB)\n",
"\u001b[K |████████████████████████████████| 11.8 MB 73.2 MB/s \n",
"\u001b[?25h Downloading pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.1 MB)\n",
"\u001b[K |████████████████████████████████| 11.1 MB 61.1 MB/s \n",
"\u001b[?25hRequirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<3dev,>=1.21.0->google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (2.10)\n",
"Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<3dev,>=1.21.0->google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (3.0.4)\n",
"Requirement already satisfied: wheel in /usr/local/lib/python3.7/dist-packages (from strip-hints<1,>=0.1.8->kfp==1.8.9->-r requirements.txt (line 1)) (0.38.3)\n",
"Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->jsonschema<4,>=3.0.1->kfp==1.8.9->-r requirements.txt (line 1)) (3.10.0)\n",
"Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib->kubernetes<19,>=8.0.0->kfp==1.8.9->-r requirements.txt (line 1)) (3.2.2)\n",
"Building wheels for collected packages: kfp, fire, kfp-server-api, strip-hints\n",
" Building wheel for kfp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
" Created wheel for kfp: filename=kfp-1.8.9-py3-none-any.whl size=409652 sha256=43e44f032c43467623798159d60362ba9ab12ac7467a89d19b5a4466995c9b69\n",
" Stored in directory: /root/.cache/pip/wheels/0e/20/7e/c2c43249eb0538c5aa2542bcc9b02affb0211ed5617fbd4abc\n",
" Building wheel for fire (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
" Created wheel for fire: filename=fire-0.4.0-py2.py3-none-any.whl size=115940 sha256=eb7248330010e4ad64864fe4fc84c7c6071e087c8600f323385fd22323e3473f\n",
" Stored in directory: /root/.cache/pip/wheels/8a/67/fb/2e8a12fa16661b9d5af1f654bd199366799740a85c64981226\n",
" Building wheel for kfp-server-api (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
" Created wheel for kfp-server-api: filename=kfp_server_api-1.8.5-py3-none-any.whl size=99714 sha256=a72a28dd2826841a3b7bf6963274d20ec0071f8196240d59bc6e977e22f45031\n",
" Stored in directory: /root/.cache/pip/wheels/77/0e/7b/ed385d69453b7b754834c01d83fa9f5708ba66b4f6ed5d6a35\n",
" Building wheel for strip-hints (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
" Created wheel for strip-hints: filename=strip_hints-0.1.10-py2.py3-none-any.whl size=22301 sha256=3bf78a6eefb5344febdc8fe4a7ca165f5e38b2b5ee816f60c3b339c19fed397d\n",
" Stored in directory: /root/.cache/pip/wheels/5e/14/c3/6e44e9b2545f2d570b03f5b6d38c00b7534aa8abb376978363\n",
"Successfully built kfp fire kfp-server-api strip-hints\n",
"Installing collected packages: cachetools, google-auth, typing-extensions, websocket-client, PyYAML, strip-hints, requests-toolbelt, pydantic, kubernetes, kfp-server-api, kfp-pipeline-spec, jsonschema, google-cloud-storage, fire, docstring-parser, Deprecated, cloudpickle, absl-py, kfp\n",
"\u001b[33m WARNING: The script wsdump is installed in '/root/.local/bin' which is not on PATH.\n",
" Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
"\u001b[33m WARNING: The script strip-hints is installed in '/root/.local/bin' which is not on PATH.\n",
" Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
"\u001b[33m WARNING: The script jsonschema is installed in '/root/.local/bin' which is not on PATH.\n",
" Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
"\u001b[33m WARNING: The scripts dsl-compile, dsl-compile-v2 and kfp are installed in '/root/.local/bin' which is not on PATH.\n",
" Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
"tensorflow 2.9.2 requires absl-py>=1.0.0, but you have absl-py 0.11.0 which is incompatible.\u001b[0m\n",
"Successfully installed Deprecated-1.2.13 PyYAML-5.4.1 absl-py-0.11.0 cachetools-4.2.4 cloudpickle-2.2.0 docstring-parser-0.15 fire-0.4.0 google-auth-1.35.0 google-cloud-storage-1.44.0 jsonschema-3.2.0 kfp-1.8.9 kfp-pipeline-spec-0.1.16 kfp-server-api-1.8.5 kubernetes-18.20.0 pydantic-1.9.2 requests-toolbelt-0.10.1 strip-hints-0.1.10 typing-extensions-3.10.0.2 websocket-client-1.4.2\n"
]
},
{
"output_type": "display_data",
"data": {
"application/vnd.colab-display-data+json": {
"pip_warning": {
"packages": [
"google",
"typing_extensions"
]
}
}
},
"metadata": {}
},
{
"output_type": "error",
"ename": "ModuleNotFoundError",
"evalue": "ignored",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-1-16b5e915de32>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtyping\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mNamedTuple\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mkfp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkfp\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdsl\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomponents\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfunc_to_container_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mInputPath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mOutputPath\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'kfp'",
"",
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
],
"errorDetails": {
"actions": [
{
"action": "open_url",
"actionText": "Open Examples",
"url": "/notebooks/snippets/importing_libraries.ipynb"
}
]
}
}
],
"source": [
"#!/usr/bin/env python\n",
"# coding: utf-8\n",
"\n",
"# In[2]:\n",
"\n",
"\n",
"with open(\"requirements.txt\", \"w\") as f:\n",
" f.write(\"kfp==1.8.9\\n\")\n",
" \n",
"get_ipython().system('pip install -r requirements.txt --upgrade --user')\n",
"\n",
"\n",
"# In[1]:\n",
"\n",
"\n",
"from typing import NamedTuple\n",
"\n",
"import kfp\n",
"from kfp import dsl\n",
"from kfp.components import func_to_container_op, InputPath, OutputPath\n",
"\n",
"from typing import NamedTuple\n",
"def train(log_folder:str) -> NamedTuple('Outputs', [('logdir', str)]):\n",
" \n",
" print('mnist_func:', log_folder)\n",
" import tensorflow as tf\n",
" import json\n",
" mnist = tf.keras.datasets.mnist\n",
" (x_train,y_train), (x_test, y_test) = mnist.load_data()\n",
" x_train, x_test = x_train/255.0, x_test/255.0\n",
"\n",
" def create_model():\n",
" return tf.keras.models.Sequential([\n",
" tf.keras.layers.Flatten(input_shape = (28,28)),\n",
" tf.keras.layers.Dense(512, activation = 'relu'),\n",
" tf.keras.layers.Dropout(0.2),\n",
" tf.keras.layers.Dense(10, activation = 'softmax')\n",
" ])\n",
" model = create_model()\n",
" model.compile(optimizer='adam',\n",
" loss='sparse_categorical_crossentropy',\n",
" metrics=['accuracy'])\n",
" import datetime\n",
" import os\n",
" \n",
" ### add tensorboard logout callback\n",
" log_dir = os.path.join(log_folder, \"logs\", datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
" tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
" ######\n",
" \n",
" model.fit(x=x_train, \n",
" y=y_train, \n",
" epochs=5, \n",
" validation_data=(x_test, y_test), \n",
" callbacks=[tensorboard_callback])\n",
"\n",
" print('At least tensorboard callbacks are correct')\n",
" print('logdir:', log_dir)\n",
" return ([log_dir])\n",
"\n",
"def prepare_tensorboard_from_localdir(pvc_name:str) -> NamedTuple('Outputs', [('mlpipeline_ui_metadata', 'UI_metadata')]):\n",
" import json\n",
" import kfp.components as components\n",
" prepare_tensorboard = components.load_component_from_url(\n",
" 'https://raw.githubusercontent.com/kubeflow/pipelines/1b107eb4bb2510ecb99fd5f4fb438cbf7c96a87a/components/contrib/tensorflow/tensorboard/prepare_tensorboard/component.yaml'\n",
")\n",
" # log_dir_uri is consisted of volume:///folders\n",
" # the volume.name should be same as ones specified in pod_template.spec.volumes.name\n",
" \n",
" return prepare_tensorboard(\n",
" log_dir_uri=f'volume://mypvc/logs', \n",
" image=\"footprintai/tensorboard:2.7.0\", \n",
" pod_template_spec=json.dumps({\n",
" \"spec\": {\n",
" \"containers\": [\n",
" {\n",
" \"volumeMounts\": [\n",
" {\n",
" \"mountPath\": \"/data\",\n",
" \"name\": \"mypvc\"\n",
" }\n",
" ]\n",
" }\n",
" ],\n",
" \"serviceAccountName\": \"default-editor\",\n",
" \"volumes\": [\n",
" {\n",
" \"name\": \"mypvc\",\n",
" \"persistentVolumeClaim\": {\n",
" \"claimName\": pvc_name\n",
" }\n",
" }\n",
" ]\n",
" }\n",
" }),\n",
" )\n",
"\n",
"def model(text1):\n",
" return dsl.ContainerOp(\n",
" name='model',\n",
" image='library/bash:4.4.23',\n",
" command=['sh', '-c'],\n",
" arguments=['echo \"$0\"', text1])\n",
"\n",
"\n",
"# In[2]:\n",
"\n",
"\n",
"import kfp.dsl as dsl\n",
"import kfp.components as components\n",
"import time\n",
"\n",
"\n",
"@dsl.pipeline(\n",
" name='tf pipeline',\n",
" description='A pipeline to train a model on tf dataset and start a tensorboard.'\n",
")\n",
"def tf_pipeline(text1='message 1'):\n",
"\n",
" log_folder = '/data'\n",
" pvc_name = 'input'\n",
" unique_pvc_resource_name = 'my-awesome-kf-workshop-%d'% int(time.time())\n",
" \n",
" vop = dsl.VolumeOp(\n",
" name=pvc_name,\n",
" resource_name=unique_pvc_resource_name,\n",
" size=\"1Gi\",\n",
" modes=dsl.VOLUME_MODE_RWO,\n",
" generate_unique_name=False,\n",
" )\n",
" tf_op = func_to_container_op(\n",
" func=train,\n",
" base_image=\"tensorflow/tensorflow:2.0.0-py3\",\n",
" )\n",
" tensorboard_task = prepare_tensorboard_from_localdir(unique_pvc_resource_name)\n",
" \n",
" tf_task = tf_op(log_folder).add_pvolumes({\n",
" log_folder:vop.volume,\n",
" })\n",
"\n",
" \n",
" \n",
" step1_task = model(text1)\n",
"# step1_task.after(tensorboard_task) \n",
" tensorboard_task.after(tf_task)\n",
" tf_task.after(step1_task)\n",
"\n",
"\n",
"# In[4]:\n",
"\n",
"\n",
"kfp.compiler.Compiler().compile(tf_pipeline, 'helloworld.zip')\n",
"\n",
"\n",
"# In[ ]:\n"
]
}
]
}