mirror of https://github.com/kubeflow/examples.git
346 lines
24 KiB
Plaintext
346 lines
24 KiB
Plaintext
{
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0,
|
|
"metadata": {
|
|
"colab": {
|
|
"provenance": []
|
|
},
|
|
"kernelspec": {
|
|
"name": "python3",
|
|
"display_name": "Python 3"
|
|
},
|
|
"language_info": {
|
|
"name": "python"
|
|
}
|
|
},
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/",
|
|
"height": 1000
|
|
},
|
|
"id": "w3cO7lQMtLkh",
|
|
"outputId": "267a2cff-6127-4946-8f3d-2ef927df4f29"
|
|
},
|
|
"outputs": [
|
|
{
|
|
"output_type": "stream",
|
|
"name": "stdout",
|
|
"text": [
|
|
"Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
|
|
"Collecting kfp==1.8.9\n",
|
|
" Downloading kfp-1.8.9.tar.gz (296 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 296 kB 20.1 MB/s \n",
|
|
"\u001b[?25hCollecting absl-py<=0.11,>=0.9\n",
|
|
" Downloading absl_py-0.11.0-py3-none-any.whl (127 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 127 kB 73.9 MB/s \n",
|
|
"\u001b[?25hCollecting PyYAML<6,>=5.3\n",
|
|
" Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 636 kB 57.3 MB/s \n",
|
|
"\u001b[?25hCollecting google-cloud-storage<2,>=1.20.0\n",
|
|
" Downloading google_cloud_storage-1.44.0-py2.py3-none-any.whl (106 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 106 kB 64.8 MB/s \n",
|
|
"\u001b[?25hCollecting kubernetes<19,>=8.0.0\n",
|
|
" Downloading kubernetes-18.20.0-py2.py3-none-any.whl (1.6 MB)\n",
|
|
"\u001b[K |████████████████████████████████| 1.6 MB 64.0 MB/s \n",
|
|
"\u001b[?25hRequirement already satisfied: google-api-python-client<2,>=1.7.8 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (1.12.11)\n",
|
|
"Collecting google-auth<2,>=1.6.1\n",
|
|
" Downloading google_auth-1.35.0-py2.py3-none-any.whl (152 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 152 kB 66.3 MB/s \n",
|
|
"\u001b[?25hCollecting requests-toolbelt<1,>=0.8.0\n",
|
|
" Downloading requests_toolbelt-0.10.1-py2.py3-none-any.whl (54 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 54 kB 3.4 MB/s \n",
|
|
"\u001b[?25hCollecting cloudpickle<3,>=2.0.0\n",
|
|
" Downloading cloudpickle-2.2.0-py3-none-any.whl (25 kB)\n",
|
|
"Collecting kfp-server-api<2.0.0,>=1.1.2\n",
|
|
" Downloading kfp-server-api-1.8.5.tar.gz (58 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 58 kB 6.1 MB/s \n",
|
|
"\u001b[?25hCollecting jsonschema<4,>=3.0.1\n",
|
|
" Downloading jsonschema-3.2.0-py2.py3-none-any.whl (56 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 56 kB 5.3 MB/s \n",
|
|
"\u001b[?25hRequirement already satisfied: tabulate<1,>=0.8.6 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (0.8.10)\n",
|
|
"Requirement already satisfied: click<9,>=7.1.2 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (7.1.2)\n",
|
|
"Collecting Deprecated<2,>=1.2.7\n",
|
|
" Downloading Deprecated-1.2.13-py2.py3-none-any.whl (9.6 kB)\n",
|
|
"Collecting strip-hints<1,>=0.1.8\n",
|
|
" Downloading strip-hints-0.1.10.tar.gz (29 kB)\n",
|
|
"Collecting docstring-parser<1,>=0.7.3\n",
|
|
" Downloading docstring_parser-0.15-py3-none-any.whl (36 kB)\n",
|
|
"Collecting kfp-pipeline-spec<0.2.0,>=0.1.13\n",
|
|
" Downloading kfp_pipeline_spec-0.1.16-py3-none-any.whl (19 kB)\n",
|
|
"Collecting fire<1,>=0.3.1\n",
|
|
" Downloading fire-0.4.0.tar.gz (87 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 87 kB 7.8 MB/s \n",
|
|
"\u001b[?25hRequirement already satisfied: protobuf<4,>=3.13.0 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (3.19.6)\n",
|
|
"Requirement already satisfied: uritemplate<4,>=3.0.1 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (3.0.1)\n",
|
|
"Requirement already satisfied: pydantic<2,>=1.8.2 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (1.10.2)\n",
|
|
"Requirement already satisfied: typer<1.0,>=0.3.2 in /usr/local/lib/python3.7/dist-packages (from kfp==1.8.9->-r requirements.txt (line 1)) (0.4.2)\n",
|
|
"Collecting typing-extensions<4,>=3.7.4\n",
|
|
" Downloading typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)\n",
|
|
"Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py<=0.11,>=0.9->kfp==1.8.9->-r requirements.txt (line 1)) (1.15.0)\n",
|
|
"Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.7/dist-packages (from Deprecated<2,>=1.2.7->kfp==1.8.9->-r requirements.txt (line 1)) (1.14.1)\n",
|
|
"Requirement already satisfied: termcolor in /usr/local/lib/python3.7/dist-packages (from fire<1,>=0.3.1->kfp==1.8.9->-r requirements.txt (line 1)) (2.1.0)\n",
|
|
"Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (0.0.4)\n",
|
|
"Requirement already satisfied: google-api-core<3dev,>=1.21.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (2.8.2)\n",
|
|
"Requirement already satisfied: httplib2<1dev,>=0.15.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (0.17.4)\n",
|
|
"Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /usr/local/lib/python3.7/dist-packages (from google-api-core<3dev,>=1.21.0->google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (1.56.4)\n",
|
|
"Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<3dev,>=1.21.0->google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (2.23.0)\n",
|
|
"Collecting cachetools<5.0,>=2.0.0\n",
|
|
" Downloading cachetools-4.2.4-py3-none-any.whl (10 kB)\n",
|
|
"Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.1->kfp==1.8.9->-r requirements.txt (line 1)) (4.9)\n",
|
|
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.1->kfp==1.8.9->-r requirements.txt (line 1)) (0.2.8)\n",
|
|
"Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.1->kfp==1.8.9->-r requirements.txt (line 1)) (57.4.0)\n",
|
|
"Requirement already satisfied: google-cloud-core<3.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.9->-r requirements.txt (line 1)) (2.3.2)\n",
|
|
"Requirement already satisfied: google-resumable-media<3.0dev,>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.9->-r requirements.txt (line 1)) (2.4.0)\n",
|
|
"Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.7/dist-packages (from google-resumable-media<3.0dev,>=1.3.0->google-cloud-storage<2,>=1.20.0->kfp==1.8.9->-r requirements.txt (line 1)) (1.5.0)\n",
|
|
"Requirement already satisfied: pyrsistent>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema<4,>=3.0.1->kfp==1.8.9->-r requirements.txt (line 1)) (0.19.2)\n",
|
|
"Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from jsonschema<4,>=3.0.1->kfp==1.8.9->-r requirements.txt (line 1)) (4.13.0)\n",
|
|
"Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema<4,>=3.0.1->kfp==1.8.9->-r requirements.txt (line 1)) (22.1.0)\n",
|
|
"Requirement already satisfied: urllib3>=1.15 in /usr/local/lib/python3.7/dist-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.9->-r requirements.txt (line 1)) (1.24.3)\n",
|
|
"Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.9->-r requirements.txt (line 1)) (2022.9.24)\n",
|
|
"Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.9->-r requirements.txt (line 1)) (2.8.2)\n",
|
|
"Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.7/dist-packages (from kubernetes<19,>=8.0.0->kfp==1.8.9->-r requirements.txt (line 1)) (1.3.1)\n",
|
|
"Collecting websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0\n",
|
|
" Downloading websocket_client-1.4.2-py3-none-any.whl (55 kB)\n",
|
|
"\u001b[K |████████████████████████████████| 55 kB 4.1 MB/s \n",
|
|
"\u001b[?25hRequirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.1->kfp==1.8.9->-r requirements.txt (line 1)) (0.4.8)\n",
|
|
"Collecting pydantic<2,>=1.8.2\n",
|
|
" Downloading pydantic-1.10.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.8 MB)\n",
|
|
"\u001b[K |████████████████████████████████| 11.8 MB 34.0 MB/s \n",
|
|
"\u001b[?25h Downloading pydantic-1.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.8 MB)\n",
|
|
"\u001b[K |████████████████████████████████| 11.8 MB 73.2 MB/s \n",
|
|
"\u001b[?25h Downloading pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.1 MB)\n",
|
|
"\u001b[K |████████████████████████████████| 11.1 MB 61.1 MB/s \n",
|
|
"\u001b[?25hRequirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<3dev,>=1.21.0->google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (2.10)\n",
|
|
"Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<3dev,>=1.21.0->google-api-python-client<2,>=1.7.8->kfp==1.8.9->-r requirements.txt (line 1)) (3.0.4)\n",
|
|
"Requirement already satisfied: wheel in /usr/local/lib/python3.7/dist-packages (from strip-hints<1,>=0.1.8->kfp==1.8.9->-r requirements.txt (line 1)) (0.38.3)\n",
|
|
"Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->jsonschema<4,>=3.0.1->kfp==1.8.9->-r requirements.txt (line 1)) (3.10.0)\n",
|
|
"Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib->kubernetes<19,>=8.0.0->kfp==1.8.9->-r requirements.txt (line 1)) (3.2.2)\n",
|
|
"Building wheels for collected packages: kfp, fire, kfp-server-api, strip-hints\n",
|
|
" Building wheel for kfp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
|
" Created wheel for kfp: filename=kfp-1.8.9-py3-none-any.whl size=409652 sha256=43e44f032c43467623798159d60362ba9ab12ac7467a89d19b5a4466995c9b69\n",
|
|
" Stored in directory: /root/.cache/pip/wheels/0e/20/7e/c2c43249eb0538c5aa2542bcc9b02affb0211ed5617fbd4abc\n",
|
|
" Building wheel for fire (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
|
" Created wheel for fire: filename=fire-0.4.0-py2.py3-none-any.whl size=115940 sha256=eb7248330010e4ad64864fe4fc84c7c6071e087c8600f323385fd22323e3473f\n",
|
|
" Stored in directory: /root/.cache/pip/wheels/8a/67/fb/2e8a12fa16661b9d5af1f654bd199366799740a85c64981226\n",
|
|
" Building wheel for kfp-server-api (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
|
" Created wheel for kfp-server-api: filename=kfp_server_api-1.8.5-py3-none-any.whl size=99714 sha256=a72a28dd2826841a3b7bf6963274d20ec0071f8196240d59bc6e977e22f45031\n",
|
|
" Stored in directory: /root/.cache/pip/wheels/77/0e/7b/ed385d69453b7b754834c01d83fa9f5708ba66b4f6ed5d6a35\n",
|
|
" Building wheel for strip-hints (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
|
" Created wheel for strip-hints: filename=strip_hints-0.1.10-py2.py3-none-any.whl size=22301 sha256=3bf78a6eefb5344febdc8fe4a7ca165f5e38b2b5ee816f60c3b339c19fed397d\n",
|
|
" Stored in directory: /root/.cache/pip/wheels/5e/14/c3/6e44e9b2545f2d570b03f5b6d38c00b7534aa8abb376978363\n",
|
|
"Successfully built kfp fire kfp-server-api strip-hints\n",
|
|
"Installing collected packages: cachetools, google-auth, typing-extensions, websocket-client, PyYAML, strip-hints, requests-toolbelt, pydantic, kubernetes, kfp-server-api, kfp-pipeline-spec, jsonschema, google-cloud-storage, fire, docstring-parser, Deprecated, cloudpickle, absl-py, kfp\n",
|
|
"\u001b[33m WARNING: The script wsdump is installed in '/root/.local/bin' which is not on PATH.\n",
|
|
" Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
|
|
"\u001b[33m WARNING: The script strip-hints is installed in '/root/.local/bin' which is not on PATH.\n",
|
|
" Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
|
|
"\u001b[33m WARNING: The script jsonschema is installed in '/root/.local/bin' which is not on PATH.\n",
|
|
" Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
|
|
"\u001b[33m WARNING: The scripts dsl-compile, dsl-compile-v2 and kfp are installed in '/root/.local/bin' which is not on PATH.\n",
|
|
" Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
|
|
"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
|
|
"tensorflow 2.9.2 requires absl-py>=1.0.0, but you have absl-py 0.11.0 which is incompatible.\u001b[0m\n",
|
|
"Successfully installed Deprecated-1.2.13 PyYAML-5.4.1 absl-py-0.11.0 cachetools-4.2.4 cloudpickle-2.2.0 docstring-parser-0.15 fire-0.4.0 google-auth-1.35.0 google-cloud-storage-1.44.0 jsonschema-3.2.0 kfp-1.8.9 kfp-pipeline-spec-0.1.16 kfp-server-api-1.8.5 kubernetes-18.20.0 pydantic-1.9.2 requests-toolbelt-0.10.1 strip-hints-0.1.10 typing-extensions-3.10.0.2 websocket-client-1.4.2\n"
|
|
]
|
|
},
|
|
{
|
|
"output_type": "display_data",
|
|
"data": {
|
|
"application/vnd.colab-display-data+json": {
|
|
"pip_warning": {
|
|
"packages": [
|
|
"google",
|
|
"typing_extensions"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"metadata": {}
|
|
},
|
|
{
|
|
"output_type": "error",
|
|
"ename": "ModuleNotFoundError",
|
|
"evalue": "ignored",
|
|
"traceback": [
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
|
|
"\u001b[0;32m<ipython-input-1-16b5e915de32>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtyping\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mNamedTuple\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mkfp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkfp\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdsl\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomponents\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfunc_to_container_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mInputPath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mOutputPath\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
|
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'kfp'",
|
|
"",
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
|
|
],
|
|
"errorDetails": {
|
|
"actions": [
|
|
{
|
|
"action": "open_url",
|
|
"actionText": "Open Examples",
|
|
"url": "/notebooks/snippets/importing_libraries.ipynb"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"source": [
|
|
"#!/usr/bin/env python\n",
|
|
"# coding: utf-8\n",
|
|
"\n",
|
|
"# In[2]:\n",
|
|
"\n",
|
|
"\n",
|
|
"with open(\"requirements.txt\", \"w\") as f:\n",
|
|
" f.write(\"kfp==1.8.9\\n\")\n",
|
|
" \n",
|
|
"get_ipython().system('pip install -r requirements.txt --upgrade --user')\n",
|
|
"\n",
|
|
"\n",
|
|
"# In[1]:\n",
|
|
"\n",
|
|
"\n",
|
|
"from typing import NamedTuple\n",
|
|
"\n",
|
|
"import kfp\n",
|
|
"from kfp import dsl\n",
|
|
"from kfp.components import func_to_container_op, InputPath, OutputPath\n",
|
|
"\n",
|
|
"from typing import NamedTuple\n",
|
|
"def train(log_folder:str) -> NamedTuple('Outputs', [('logdir', str)]):\n",
|
|
" \n",
|
|
" print('mnist_func:', log_folder)\n",
|
|
" import tensorflow as tf\n",
|
|
" import json\n",
|
|
" mnist = tf.keras.datasets.mnist\n",
|
|
" (x_train,y_train), (x_test, y_test) = mnist.load_data()\n",
|
|
" x_train, x_test = x_train/255.0, x_test/255.0\n",
|
|
"\n",
|
|
" def create_model():\n",
|
|
" return tf.keras.models.Sequential([\n",
|
|
" tf.keras.layers.Flatten(input_shape = (28,28)),\n",
|
|
" tf.keras.layers.Dense(512, activation = 'relu'),\n",
|
|
" tf.keras.layers.Dropout(0.2),\n",
|
|
" tf.keras.layers.Dense(10, activation = 'softmax')\n",
|
|
" ])\n",
|
|
" model = create_model()\n",
|
|
" model.compile(optimizer='adam',\n",
|
|
" loss='sparse_categorical_crossentropy',\n",
|
|
" metrics=['accuracy'])\n",
|
|
" import datetime\n",
|
|
" import os\n",
|
|
" \n",
|
|
" ### add tensorboard logout callback\n",
|
|
" log_dir = os.path.join(log_folder, \"logs\", datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
|
|
" tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
|
|
" ######\n",
|
|
" \n",
|
|
" model.fit(x=x_train, \n",
|
|
" y=y_train, \n",
|
|
" epochs=5, \n",
|
|
" validation_data=(x_test, y_test), \n",
|
|
" callbacks=[tensorboard_callback])\n",
|
|
"\n",
|
|
" print('At least tensorboard callbacks are correct')\n",
|
|
" print('logdir:', log_dir)\n",
|
|
" return ([log_dir])\n",
|
|
"\n",
|
|
"def prepare_tensorboard_from_localdir(pvc_name:str) -> NamedTuple('Outputs', [('mlpipeline_ui_metadata', 'UI_metadata')]):\n",
|
|
" import json\n",
|
|
" import kfp.components as components\n",
|
|
" prepare_tensorboard = components.load_component_from_url(\n",
|
|
" 'https://raw.githubusercontent.com/kubeflow/pipelines/1b107eb4bb2510ecb99fd5f4fb438cbf7c96a87a/components/contrib/tensorflow/tensorboard/prepare_tensorboard/component.yaml'\n",
|
|
")\n",
|
|
" # log_dir_uri is consisted of volume:///folders\n",
|
|
" # the volume.name should be same as ones specified in pod_template.spec.volumes.name\n",
|
|
" \n",
|
|
" return prepare_tensorboard(\n",
|
|
" log_dir_uri=f'volume://mypvc/logs', \n",
|
|
" image=\"footprintai/tensorboard:2.7.0\", \n",
|
|
" pod_template_spec=json.dumps({\n",
|
|
" \"spec\": {\n",
|
|
" \"containers\": [\n",
|
|
" {\n",
|
|
" \"volumeMounts\": [\n",
|
|
" {\n",
|
|
" \"mountPath\": \"/data\",\n",
|
|
" \"name\": \"mypvc\"\n",
|
|
" }\n",
|
|
" ]\n",
|
|
" }\n",
|
|
" ],\n",
|
|
" \"serviceAccountName\": \"default-editor\",\n",
|
|
" \"volumes\": [\n",
|
|
" {\n",
|
|
" \"name\": \"mypvc\",\n",
|
|
" \"persistentVolumeClaim\": {\n",
|
|
" \"claimName\": pvc_name\n",
|
|
" }\n",
|
|
" }\n",
|
|
" ]\n",
|
|
" }\n",
|
|
" }),\n",
|
|
" )\n",
|
|
"\n",
|
|
"def model(text1):\n",
|
|
" return dsl.ContainerOp(\n",
|
|
" name='model',\n",
|
|
" image='library/bash:4.4.23',\n",
|
|
" command=['sh', '-c'],\n",
|
|
" arguments=['echo \"$0\"', text1])\n",
|
|
"\n",
|
|
"\n",
|
|
"# In[2]:\n",
|
|
"\n",
|
|
"\n",
|
|
"import kfp.dsl as dsl\n",
|
|
"import kfp.components as components\n",
|
|
"import time\n",
|
|
"\n",
|
|
"\n",
|
|
"@dsl.pipeline(\n",
|
|
" name='tf pipeline',\n",
|
|
" description='A pipeline to train a model on tf dataset and start a tensorboard.'\n",
|
|
")\n",
|
|
"def tf_pipeline(text1='message 1'):\n",
|
|
"\n",
|
|
" log_folder = '/data'\n",
|
|
" pvc_name = 'input'\n",
|
|
" unique_pvc_resource_name = 'my-awesome-kf-workshop-%d'% int(time.time())\n",
|
|
" \n",
|
|
" vop = dsl.VolumeOp(\n",
|
|
" name=pvc_name,\n",
|
|
" resource_name=unique_pvc_resource_name,\n",
|
|
" size=\"1Gi\",\n",
|
|
" modes=dsl.VOLUME_MODE_RWO,\n",
|
|
" generate_unique_name=False,\n",
|
|
" )\n",
|
|
" tf_op = func_to_container_op(\n",
|
|
" func=train,\n",
|
|
" base_image=\"tensorflow/tensorflow:2.0.0-py3\",\n",
|
|
" )\n",
|
|
" tensorboard_task = prepare_tensorboard_from_localdir(unique_pvc_resource_name)\n",
|
|
" \n",
|
|
" tf_task = tf_op(log_folder).add_pvolumes({\n",
|
|
" log_folder:vop.volume,\n",
|
|
" })\n",
|
|
"\n",
|
|
" \n",
|
|
" \n",
|
|
" step1_task = model(text1)\n",
|
|
"# step1_task.after(tensorboard_task) \n",
|
|
" tensorboard_task.after(tf_task)\n",
|
|
" tf_task.after(step1_task)\n",
|
|
"\n",
|
|
"\n",
|
|
"# In[4]:\n",
|
|
"\n",
|
|
"\n",
|
|
"kfp.compiler.Compiler().compile(tf_pipeline, 'helloworld.zip')\n",
|
|
"\n",
|
|
"\n",
|
|
"# In[ ]:\n"
|
|
]
|
|
}
|
|
]
|
|
} |