diff --git a/xgboost_synthetic/build-train-deploy.ipynb b/xgboost_synthetic/build-train-deploy.ipynb index 14b39699..144c5663 100644 --- a/xgboost_synthetic/build-train-deploy.ipynb +++ b/xgboost_synthetic/build-train-deploy.ipynb @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -56,20 +56,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: retrying in /home/jovyan/.local/lib/python3.6/site-packages (1.3.3)\n", - "Requirement already satisfied: six>=1.7.0 in /opt/conda/lib/python3.6/site-packages (from retrying) (1.12.0)\n", - "\u001b[33mYou are using pip version 19.0.1, however version 19.2.2 is available.\n", - "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "!pip3 install retrying" ] @@ -78,108 +67,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "* Install a specific version of fairing that this example is tested against" + "* Install a specific version of kubeflow-fairing that this example is tested against" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting git+git://github.com/kubeflow/fairing.git@c6c075dece72135f5883abfe2a296894d74a2367\n", - " Cloning git://github.com/kubeflow/fairing.git (to revision c6c075dece72135f5883abfe2a296894d74a2367) to /tmp/pip-req-build-n8eh3_5o\n", - "Requirement already satisfied (use --upgrade to upgrade): fairing==0.5.3 from git+git://github.com/kubeflow/fairing.git@c6c075dece72135f5883abfe2a296894d74a2367 in /opt/conda/lib/python3.6/site-packages\n", - "Requirement already satisfied: docker>=3.4.1 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (3.7.2)\n", - "Requirement already satisfied: notebook>=5.6.0 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (5.7.8)\n", - "Requirement already satisfied: kubernetes>=9.0.0 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (9.0.0)\n", - "Requirement already satisfied: future>=0.17.1 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (0.17.1)\n", - "Requirement already satisfied: six>=1.11.0 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (1.12.0)\n", - "Requirement already satisfied: google-cloud-storage>=1.13.2 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (1.14.0)\n", - "Requirement already satisfied: requests>=2.21.0 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (2.21.0)\n", - "Requirement already satisfied: setuptools>=34.0.0 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (40.9.0)\n", - "Requirement already satisfied: google-auth>=1.6.2 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (1.6.3)\n", - "Requirement already satisfied: httplib2>=0.12.0 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (0.12.1)\n", - "Requirement already satisfied: oauth2client>=4.0.0 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (4.1.3)\n", - "Requirement already satisfied: tornado<6.0.0,>=5.1.1 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (5.1.1)\n", - "Requirement already satisfied: google-api-python-client>=1.7.8 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (1.7.8)\n", - "Requirement already satisfied: cloudpickle>=0.8 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (0.8.1)\n", - "Requirement already satisfied: numpy>=1.14 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (1.16.2)\n", - "Requirement already satisfied: urllib3==1.24.2 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (1.24.2)\n", - "Requirement already satisfied: boto3>=1.9.0 in /opt/conda/lib/python3.6/site-packages (from fairing==0.5.3) (1.9.210)\n", - "Requirement already satisfied: docker-pycreds>=0.4.0 in /opt/conda/lib/python3.6/site-packages (from docker>=3.4.1->fairing==0.5.3) (0.4.0)\n", - "Requirement already satisfied: websocket-client>=0.32.0 in /opt/conda/lib/python3.6/site-packages (from docker>=3.4.1->fairing==0.5.3) (0.56.0)\n", - "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (2.10)\n", - "Requirement already satisfied: traitlets>=4.2.1 in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (4.3.2)\n", - "Requirement already satisfied: ipython-genutils in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (0.2.0)\n", - "Requirement already satisfied: pyzmq>=17 in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (18.0.1)\n", - "Requirement already satisfied: jupyter-core>=4.4.0 in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (4.4.0)\n", - "Requirement already satisfied: jupyter-client>=5.2.0 in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (5.2.4)\n", - "Requirement already satisfied: nbconvert in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (5.4.1)\n", - "Requirement already satisfied: Send2Trash in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (1.5.0)\n", - "Requirement already satisfied: nbformat in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (4.4.0)\n", - "Requirement already satisfied: terminado>=0.8.1 in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (0.8.2)\n", - "Requirement already satisfied: ipykernel in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (5.1.0)\n", - "Requirement already satisfied: prometheus-client in /opt/conda/lib/python3.6/site-packages (from notebook>=5.6.0->fairing==0.5.3) (0.6.0)\n", - "Requirement already satisfied: certifi>=14.05.14 in /opt/conda/lib/python3.6/site-packages (from kubernetes>=9.0.0->fairing==0.5.3) (2019.3.9)\n", - "Requirement already satisfied: python-dateutil>=2.5.3 in /opt/conda/lib/python3.6/site-packages (from kubernetes>=9.0.0->fairing==0.5.3) (2.8.0)\n", - "Requirement already satisfied: pyyaml>=3.12 in /opt/conda/lib/python3.6/site-packages (from kubernetes>=9.0.0->fairing==0.5.3) (5.1)\n", - "Requirement already satisfied: requests-oauthlib in /opt/conda/lib/python3.6/site-packages (from kubernetes>=9.0.0->fairing==0.5.3) (1.2.0)\n", - "Requirement already satisfied: google-api-core<2.0.0dev,>=1.6.0 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage>=1.13.2->fairing==0.5.3) (1.9.0)\n", - "Requirement already satisfied: google-resumable-media>=0.3.1 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage>=1.13.2->fairing==0.5.3) (0.3.2)\n", - "Requirement already satisfied: google-cloud-core<0.30dev,>=0.29.0 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage>=1.13.2->fairing==0.5.3) (0.29.1)\n", - "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests>=2.21.0->fairing==0.5.3) (3.0.4)\n", - "Requirement already satisfied: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests>=2.21.0->fairing==0.5.3) (2.8)\n", - "Requirement already satisfied: cachetools>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.6.2->fairing==0.5.3) (3.1.0)\n", - "Requirement already satisfied: rsa>=3.1.4 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.6.2->fairing==0.5.3) (4.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.6.2->fairing==0.5.3) (0.2.4)\n", - "Requirement already satisfied: pyasn1>=0.1.7 in /opt/conda/lib/python3.6/site-packages (from oauth2client>=4.0.0->fairing==0.5.3) (0.4.5)\n", - "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client>=1.7.8->fairing==0.5.3) (0.0.3)\n", - "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /opt/conda/lib/python3.6/site-packages (from google-api-python-client>=1.7.8->fairing==0.5.3) (3.0.0)\n", - "Requirement already satisfied: botocore<1.13.0,>=1.12.210 in /opt/conda/lib/python3.6/site-packages (from boto3>=1.9.0->fairing==0.5.3) (1.12.210)\n", - "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /opt/conda/lib/python3.6/site-packages (from boto3>=1.9.0->fairing==0.5.3) (0.2.1)\n", - "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /opt/conda/lib/python3.6/site-packages (from boto3>=1.9.0->fairing==0.5.3) (0.9.4)\n", - "Requirement already satisfied: MarkupSafe>=0.23 in /opt/conda/lib/python3.6/site-packages (from jinja2->notebook>=5.6.0->fairing==0.5.3) (1.1.1)\n", - "Requirement already satisfied: decorator in /opt/conda/lib/python3.6/site-packages (from traitlets>=4.2.1->notebook>=5.6.0->fairing==0.5.3) (4.4.0)\n", - "Requirement already satisfied: defusedxml in /opt/conda/lib/python3.6/site-packages (from nbconvert->notebook>=5.6.0->fairing==0.5.3) (0.5.0)\n", - "Requirement already satisfied: testpath in /opt/conda/lib/python3.6/site-packages (from nbconvert->notebook>=5.6.0->fairing==0.5.3) (0.4.2)\n", - "Requirement already satisfied: entrypoints>=0.2.2 in /opt/conda/lib/python3.6/site-packages (from nbconvert->notebook>=5.6.0->fairing==0.5.3) (0.3)\n", - "Requirement already satisfied: pygments in /opt/conda/lib/python3.6/site-packages (from nbconvert->notebook>=5.6.0->fairing==0.5.3) (2.3.1)\n", - "Requirement already satisfied: mistune>=0.8.1 in /opt/conda/lib/python3.6/site-packages (from nbconvert->notebook>=5.6.0->fairing==0.5.3) (0.8.4)\n", - "Requirement already satisfied: bleach in /opt/conda/lib/python3.6/site-packages (from nbconvert->notebook>=5.6.0->fairing==0.5.3) (3.1.0)\n", - "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/conda/lib/python3.6/site-packages (from nbconvert->notebook>=5.6.0->fairing==0.5.3) (1.4.2)\n", - "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /opt/conda/lib/python3.6/site-packages (from nbformat->notebook>=5.6.0->fairing==0.5.3) (3.0.1)\n", - "Requirement already satisfied: ptyprocess; os_name != \"nt\" in /opt/conda/lib/python3.6/site-packages (from terminado>=0.8.1->notebook>=5.6.0->fairing==0.5.3) (0.6.0)\n", - "Requirement already satisfied: ipython>=5.0.0 in /opt/conda/lib/python3.6/site-packages (from ipykernel->notebook>=5.6.0->fairing==0.5.3) (7.4.0)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.6/site-packages (from requests-oauthlib->kubernetes>=9.0.0->fairing==0.5.3) (3.0.1)\n", - "Requirement already satisfied: pytz in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage>=1.13.2->fairing==0.5.3) (2018.9)\n", - "Requirement already satisfied: protobuf>=3.4.0 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage>=1.13.2->fairing==0.5.3) (3.7.1)\n", - "Requirement already satisfied: googleapis-common-protos!=1.5.4,<2.0dev,>=1.5.3 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage>=1.13.2->fairing==0.5.3) (1.5.9)\n", - "Requirement already satisfied: docutils<0.16,>=0.10 in /opt/conda/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.210->boto3>=1.9.0->fairing==0.5.3) (0.15.2)\n", - "Requirement already satisfied: webencodings in /opt/conda/lib/python3.6/site-packages (from bleach->nbconvert->notebook>=5.6.0->fairing==0.5.3) (0.5.1)\n", - "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat->notebook>=5.6.0->fairing==0.5.3) (19.1.0)\n", - "Requirement already satisfied: pyrsistent>=0.14.0 in /opt/conda/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat->notebook>=5.6.0->fairing==0.5.3) (0.14.11)\n", - "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->notebook>=5.6.0->fairing==0.5.3) (2.0.9)\n", - "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /opt/conda/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->notebook>=5.6.0->fairing==0.5.3) (4.6.0)\n", - "Requirement already satisfied: backcall in /opt/conda/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->notebook>=5.6.0->fairing==0.5.3) (0.1.0)\n", - "Requirement already satisfied: jedi>=0.10 in /opt/conda/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->notebook>=5.6.0->fairing==0.5.3) (0.13.3)\n", - "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->notebook>=5.6.0->fairing==0.5.3) (0.7.5)\n", - "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.6/site-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython>=5.0.0->ipykernel->notebook>=5.6.0->fairing==0.5.3) (0.1.7)\n", - "Requirement already satisfied: parso>=0.3.0 in /opt/conda/lib/python3.6/site-packages (from jedi>=0.10->ipython>=5.0.0->ipykernel->notebook>=5.6.0->fairing==0.5.3) (0.4.0)\n", - "Building wheels for collected packages: fairing\n", - " Building wheel for fairing (setup.py) ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /tmp/pip-ephem-wheel-cache-nw9r0vn2/wheels/bf/3f/0f/c65ae27dc4acd9443a98bfe546ad571e9a2d9f05905274ae44\n", - "Successfully built fairing\n", - "\u001b[33mYou are using pip version 19.0.1, however version 19.2.2 is available.\n", - "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ - "!pip3 install git+git://github.com/kubeflow/fairing.git@c6c075dece72135f5883abfe2a296894d74a2367" + "!pip3 install git+git://github.com/kubeflow/fairing.git@840c272c11452696e4be8e2e5d8f547949da4dc7" ] }, { @@ -191,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "scrolled": false }, @@ -209,12 +106,12 @@ "metadata": {}, "source": [ "* Import the python libraries we will use\n", - "* We add a comment \"fairing:include-cell\" to tell the fairing preprocessor to keep this cell when converting to python code later" + "* We add a comment \"fairing:include-cell\" to tell the kubefow fairing preprocessor to keep this cell when converting to python code later" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -246,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -257,10 +154,10 @@ "import kfp.dsl as dsl\n", "import kfp.compiler as compiler\n", "from kubernetes import client as k8s_client\n", - "import fairing \n", - "from fairing.builders import append\n", - "from fairing.deployers import job\n", - "from fairing.preprocessors.converted_notebook import ConvertNotebookPreprocessorWithFire\n" + "from kubeflow import fairing \n", + "from kubeflow.fairing.builders import append\n", + "from kubeflow.fairing.deployers import job\n", + "from kubeflow.fairing.preprocessors.converted_notebook import ConvertNotebookPreprocessorWithFire\n" ] }, { @@ -275,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -298,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -366,12 +263,12 @@ "## Wrap Training and Prediction in a class\n", "\n", "* In the cell below we wrap training and prediction in a class\n", - "* A class provides the structure we will need to eventually use fairing to launch separate training jobs and/or deploy the model on Kubernetes" + "* A class provides the structure we will need to eventually use kubeflow fairing to launch separate training jobs and/or deploy the model on Kubernetes" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -488,83 +385,9 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "model_file=mockup-model.dat\n", - "[0]\tvalidation_0-rmse:112.335\n", - "Will train until validation_0-rmse hasn't improved in 40 rounds.\n", - "[1]\tvalidation_0-rmse:106.517\n", - "[2]\tvalidation_0-rmse:101.393\n", - "[3]\tvalidation_0-rmse:95.7595\n", - "[4]\tvalidation_0-rmse:91.8237\n", - "[5]\tvalidation_0-rmse:88.2375\n", - "[6]\tvalidation_0-rmse:83.3448\n", - "[7]\tvalidation_0-rmse:80.6862\n", - "[8]\tvalidation_0-rmse:77.7361\n", - "[9]\tvalidation_0-rmse:75.0139\n", - "[10]\tvalidation_0-rmse:72.0667\n", - "[11]\tvalidation_0-rmse:70.2466\n", - "[12]\tvalidation_0-rmse:68.9596\n", - "[13]\tvalidation_0-rmse:66.8088\n", - "[14]\tvalidation_0-rmse:64.6703\n", - "[15]\tvalidation_0-rmse:63.0301\n", - "[16]\tvalidation_0-rmse:61.259\n", - "[17]\tvalidation_0-rmse:59.987\n", - "[18]\tvalidation_0-rmse:58.5032\n", - "[19]\tvalidation_0-rmse:57.2869\n", - "[20]\tvalidation_0-rmse:56.1678\n", - "[21]\tvalidation_0-rmse:55.3422\n", - "[22]\tvalidation_0-rmse:54.2746\n", - "[23]\tvalidation_0-rmse:53.4287\n", - "[24]\tvalidation_0-rmse:52.5733\n", - "[25]\tvalidation_0-rmse:51.594\n", - "[26]\tvalidation_0-rmse:50.6855\n", - "[27]\tvalidation_0-rmse:50.0631\n", - "[28]\tvalidation_0-rmse:49.4712\n", - "[29]\tvalidation_0-rmse:48.9603\n", - "[30]\tvalidation_0-rmse:48.5284\n", - "[31]\tvalidation_0-rmse:47.9727\n", - "[32]\tvalidation_0-rmse:47.2481\n", - "[33]\tvalidation_0-rmse:46.5567\n", - "[34]\tvalidation_0-rmse:46.1378\n", - "[35]\tvalidation_0-rmse:45.2184\n", - "[36]\tvalidation_0-rmse:44.5527\n", - "[37]\tvalidation_0-rmse:44.2344\n", - "[38]\tvalidation_0-rmse:43.9775\n", - "[39]\tvalidation_0-rmse:43.398\n", - "[40]\tvalidation_0-rmse:43.0096\n", - "[41]\tvalidation_0-rmse:42.3342\n", - "[42]\tvalidation_0-rmse:42.0125\n", - "[43]\tvalidation_0-rmse:41.7512\n", - "[44]\tvalidation_0-rmse:41.3573\n", - "[45]\tvalidation_0-rmse:41.1225\n", - "[46]\tvalidation_0-rmse:40.9997\n", - "[47]\tvalidation_0-rmse:40.736\n", - "[48]\tvalidation_0-rmse:40.498\n", - "[49]\tvalidation_0-rmse:40.0949\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "mean_absolute_error=31.81\n", - "Model export success: mockup-model.dat\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best RMSE on eval: %.2f with %d rounds 40.094906 50\n" - ] - } - ], + "outputs": [], "source": [ "model = ModelServe(model_file=\"mockup-model.dat\")\n", "model.train()" @@ -582,28 +405,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "model_file not supplied; using the default\n", - "model_file=mockup-model.dat\n" - ] - }, - { - "data": { - "text/plain": [ - "[[14.064239501953125, 11.37496566772461]]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(train_X, train_y), (test_X, test_y) =read_synthetic_input()\n", "\n", @@ -614,9 +418,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Use Fairing to Launch a K8s Job to train your model\n", + "## Use Kubeflow Fairing to Launch a K8s Job to train your model\n", "\n", - "* Now that we have trained a model locally we can use fairing to\n", + "* Now that we have trained a model locally we can use Kubeflow fairing to\n", " 1. Launch a Kubernetes job to train the model\n", " 1. Deploy the model on Kubernetes\n", "* Launching a separate Kubernetes job to train the model has the following advantages\n", @@ -629,16 +433,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Configure The Docker Registry For Fairing\n", + "### Configure The Docker Registry For Kubeflow Fairing\n", "\n", "* In order to build docker images from your notebook we need a docker registry where the images will be stored\n", "* Below you set some variables specifying a [GCR container registry](https://cloud.google.com/container-registry/docs/)\n", - "* Fairing provides a utility function to guess the name of your GCP project" + "* Kubeflow Fairing provides a utility function to guess the name of your GCP project" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -652,46 +456,24 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Use fairing to build the docker image\n", + "## Use Kubeflow fairing to build the docker image\n", "\n", - "* First you will use fairing's kaniko builder to build a docker image that includes all your dependencies\n", + "* First you will use kubeflow fairing's kaniko builder to build a docker image that includes all your dependencies\n", " * You use kaniko because you want to be able to run `pip` to install dependencies\n", " * Kaniko gives you the flexibility to build images from Dockerfiles\n", "* kaniko, however, can be slow\n", "* so you will build a base image using Kaniko and then every time your code changes you will just build an image\n", " starting from your base image and adding your code to it\n", - "* you use the fairing build to enable these fast rebuilds" + "* you use the kubeflow fairing build to enable these fast rebuilds" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Converting build-train-deploy.ipynb to build-train-deploy.py\n", - "Creating entry point for the class name ModelServe\n" - ] - }, - { - "data": { - "text/plain": [ - "[PosixPath('build-train-deploy.py'),\n", - " 'requirements.txt',\n", - " 'xgboost_util.py',\n", - " 'mockup-model.dat']" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "from fairing.builders import cluster\n", + "from kubeflow.fairing.builders import cluster\n", "preprocessor = ConvertNotebookPreprocessorWithFire(class_name='ModelServe', notebook_file='build-train-deploy.ipynb')\n", "\n", "if not preprocessor.input_files:\n", @@ -716,90 +498,11 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Building image using cluster builder.\n", - "Creating docker context: /tmp/fairing_context_vn0azn4y\n", - "Converting build-train-deploy.ipynb to build-train-deploy.py\n", - "Creating entry point for the class name ModelServe\n", - "Waiting for fairing-builder-ns5b5 to start...\n", - "Waiting for fairing-builder-ns5b5 to start...\n", - "Waiting for fairing-builder-ns5b5 to start...\n", - "Waiting for fairing-builder-ns5b5 to start...\n", - "Pod started running True\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36mINFO\u001b[0m[0006] Downloading base image gcr.io/kubeflow-images-public/tensorflow-1.13.1-notebook-cpu:v0.5.0\n", - "\u001b[36mINFO\u001b[0m[0006] Downloading base image gcr.io/kubeflow-images-public/tensorflow-1.13.1-notebook-cpu:v0.5.0\n", - "\u001b[33mWARN\u001b[0m[0006] Error while retrieving image from cache: getting image from path: open /cache/sha256:5aaccf0267f085afd976342a8e943a9c6cefccef5b554df4e15fa7bf15cbd7a3: no such file or directory\n", - "\u001b[36mINFO\u001b[0m[0007] Using files from context: [/kaniko/buildcontext/app/requirements.txt]\n", - "\u001b[36mINFO\u001b[0m[0007] Checking for cached layer gcr.io/jlewi-dev/fairing-job/fairing-job/cache:488afeebb3bcde81c38dc99f3de2f4ba06b85302f8440b625e8fe1b08c53ef72...\n", - "\u001b[36mINFO\u001b[0m[0007] Using caching version of cmd: RUN if [ -e requirements.txt ];then pip install --no-cache -r requirements.txt; fi\n", - "\u001b[36mINFO\u001b[0m[0007] Using files from context: [/kaniko/buildcontext/app]\n", - "\u001b[36mINFO\u001b[0m[0007] Taking snapshot of full filesystem...\n", - "\u001b[36mINFO\u001b[0m[0007] Skipping paths under /dev, as it is a whitelisted directory\n", - "\u001b[36mINFO\u001b[0m[0007] Skipping paths under /etc/secrets, as it is a whitelisted directory\n", - "\u001b[36mINFO\u001b[0m[0007] Skipping paths under /kaniko, as it is a whitelisted directory\n", - "\u001b[36mINFO\u001b[0m[0007] Skipping paths under /proc, as it is a whitelisted directory\n", - "\u001b[36mINFO\u001b[0m[0007] Skipping paths under /sys, as it is a whitelisted directory\n", - "\u001b[36mINFO\u001b[0m[0007] Skipping paths under /var/run, as it is a whitelisted directory\n", - "\u001b[36mINFO\u001b[0m[0007] WORKDIR /app/\n", - "\u001b[36mINFO\u001b[0m[0007] cmd: workdir\n", - "\u001b[36mINFO\u001b[0m[0007] Changed working directory to /app/\n", - "\u001b[36mINFO\u001b[0m[0007] Creating directory /app/\n", - "\u001b[36mINFO\u001b[0m[0007] Taking snapshot of files...\n", - "\u001b[36mINFO\u001b[0m[0007] ENV FAIRING_RUNTIME 1\n", - "\u001b[36mINFO\u001b[0m[0007] No files changed in this command, skipping snapshotting.\n", - "\u001b[36mINFO\u001b[0m[0007] Using files from context: [/kaniko/buildcontext/app/requirements.txt]\n", - "\u001b[36mINFO\u001b[0m[0007] COPY /app//requirements.txt /app/\n", - "\u001b[36mINFO\u001b[0m[0007] Taking snapshot of files...\n", - "\u001b[36mINFO\u001b[0m[0007] RUN if [ -e requirements.txt ];then pip install --no-cache -r requirements.txt; fi\n", - "\u001b[36mINFO\u001b[0m[0007] Found cached layer, extracting to filesystem\n", - "\u001b[36mINFO\u001b[0m[0010] Taking snapshot of files...\n", - "\u001b[36mINFO\u001b[0m[0013] Using files from context: [/kaniko/buildcontext/app]\n", - "\u001b[36mINFO\u001b[0m[0013] COPY /app/ /app/\n", - "\u001b[36mINFO\u001b[0m[0013] Taking snapshot of files...\n", - "2019/08/18 00:32:59 existing blob: sha256:86db56dbcdfc4e5ba205e00f3de178548dd0fcd3d1d9ec011747ca0bb08a8177\n", - "2019/08/18 00:32:59 existing blob: sha256:afde35469481d2bc446d649a7a3d099147bbf7696b66333e76a411686b617ea1\n", - "2019/08/18 00:32:59 existing blob: sha256:398d32b153e84fe343f0c5b07d65e89b05551aae6cb8b3a03bb2b662976eb3b8\n", - "2019/08/18 00:32:59 existing blob: sha256:c451d20886c33c47dab7b01b05ece292ee5173a9a4aced925035401a6b1de62e\n", - "2019/08/18 00:32:59 existing blob: sha256:124c757242f88002a858c23fc79f8262f9587fa30fd92507e586ad074afb42b6\n", - "2019/08/18 00:32:59 existing blob: sha256:47956fc6abae87d70180bc4f0efdad014b8e2a3b617a447ac01f674336737dfc\n", - "2019/08/18 00:32:59 existing blob: sha256:fa3f2f277e67c5cbbf1dac21dc27111a60d3cd2ef494d94aa1515d3319f2a245\n", - "2019/08/18 00:32:59 existing blob: sha256:432f7fba907384de9a5c1c23aed93fa3eff7d6a8d89a91f5eab99f41aa889323\n", - "2019/08/18 00:32:59 existing blob: sha256:147c5bbff888fc9cddffd4078daa35bba0d1d6f6c7175a1acb144412a43b3fce\n", - "2019/08/18 00:32:59 existing blob: sha256:969fc9c5501e60432ca0bc4b635493feb2f90e14822d2f3e3f79742fed96757d\n", - "2019/08/18 00:32:59 existing blob: sha256:167108358fe643eea57fc595ff9b76a1a7e09e022c84d724346ce5b41d0148bc\n", - "2019/08/18 00:32:59 existing blob: sha256:8485e620dff15e8a69076ac02f6b23ffb3408161cdc2c0572905838765a84854\n", - "2019/08/18 00:32:59 existing blob: sha256:547e89bdafacadd9655a394a9d73c49c9890233c0cd244cbc5b1cb859be1395c\n", - "2019/08/18 00:32:59 existing blob: sha256:59951887a0c1d1a227f43219b3bc84562a6f2a7e0ab5c276fbd9eaba6ebec02d\n", - "2019/08/18 00:32:59 existing blob: sha256:ff51e784988b3a953df5d6ba36b982436c2b16a77eb081ce7a589ca67d04144c\n", - "2019/08/18 00:32:59 existing blob: sha256:9ad0c8331ed7f0f76b54d8e91e66661a3ca35e02a25cc83ccb48d51fa89e5573\n", - "2019/08/18 00:32:59 existing blob: sha256:9d866f8bde2a0d607a6d17edc0fbd5e00b58306efc2b0a57e0ba72f269e7c6be\n", - "2019/08/18 00:32:59 existing blob: sha256:9ab35225e174496943b6a86bf62d004409479cf722ef1d3e01ca48afc8cfaa79\n", - "2019/08/18 00:32:59 existing blob: sha256:22ea01b3a354ebdcf4386e6d2f53b6cf65bd9cdcb34a70f32e00b90a477589d0\n", - "2019/08/18 00:32:59 existing blob: sha256:62228d5c51598033083adbf71e8ee3d8d523d7d6d8c9d789b8c8a2d71ca988ac\n", - "2019/08/18 00:32:59 existing blob: sha256:bd5e67bf2947497b4a4347d2751797d6b3a40f0dc5d355185815ee6da1b8ae0c\n", - "2019/08/18 00:33:01 pushed blob sha256:1d9677f30f0f4a3989f410d16a6829b3953729bb1971eab37e89535308f9bf0a\n", - "2019/08/18 00:33:01 pushed blob sha256:d76c933128c526784913ade49f5bb957fcdbf340eac405af17aaa01cb9c21650\n", - "2019/08/18 00:33:01 pushed blob sha256:9e6ab0086905f54f7c11a49bff522f9b9f8a7b85f6837e1ab9db5c13abee190e\n", - "2019/08/18 00:33:01 pushed blob sha256:a11325fad344d6f0b31cd2d1c0ac2f693faf6b7724fe43b07f42ce3e51bac3f2\n", - "2019/08/18 00:33:05 pushed blob sha256:c5aa14b34cb84e23688babb6c0c39c1591a7dfd7660c719b9d63bdcb8d2aa172\n", - "2019/08/18 00:33:05 gcr.io/jlewi-dev/fairing-job/fairing-job:A806F356: digest: sha256:5998eae5f15aa375edf498cc58f2cd0a701e80ad766b2c6c707e1166db889e35 size: 4325\n" - ] - } - ], + "outputs": [], "source": [ "# Use a stock jupyter image as our base image\n", "base_image = \"gcr.io/kubeflow-images-public/tensorflow-1.13.1-notebook-cpu:v0.5.0\"\n", @@ -836,63 +539,11 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Converting build-train-deploy.ipynb to build-train-deploy.py\n", - "Creating entry point for the class name ModelServe\n", - "Building image using Append builder...\n", - "Creating docker context: /tmp/fairing_context_v25n3350\n", - "Converting build-train-deploy.ipynb to build-train-deploy.py\n", - "Creating entry point for the class name ModelServe\n", - "build-train-deploy.py already exists in Fairing context, skipping...\n", - "Loading Docker credentials for repository 'gcr.io/jlewi-dev/fairing-job/fairing-job:A806F356'\n", - "Invoking 'docker-credential-gcloud' to obtain Docker credentials.\n", - "Successfully obtained Docker credentials.\n", - "Image successfully built in 1.0125336939818226s.\n", - "Pushing image gcr.io/jlewi-dev/fairing-job/fairing-job:A84D11BF...\n", - "Loading Docker credentials for repository 'gcr.io/jlewi-dev/fairing-job/fairing-job:A84D11BF'\n", - "Invoking 'docker-credential-gcloud' to obtain Docker credentials.\n", - "Successfully obtained Docker credentials.\n", - "Uploading gcr.io/jlewi-dev/fairing-job/fairing-job:A84D11BF\n", - "Layer sha256:8485e620dff15e8a69076ac02f6b23ffb3408161cdc2c0572905838765a84854 exists, skipping\n", - "Layer sha256:124c757242f88002a858c23fc79f8262f9587fa30fd92507e586ad074afb42b6 exists, skipping\n", - "Layer sha256:9ad0c8331ed7f0f76b54d8e91e66661a3ca35e02a25cc83ccb48d51fa89e5573 exists, skipping\n", - "Layer sha256:432f7fba907384de9a5c1c23aed93fa3eff7d6a8d89a91f5eab99f41aa889323 exists, skipping\n", - "Layer sha256:9d866f8bde2a0d607a6d17edc0fbd5e00b58306efc2b0a57e0ba72f269e7c6be exists, skipping\n", - "Layer sha256:9e6ab0086905f54f7c11a49bff522f9b9f8a7b85f6837e1ab9db5c13abee190e exists, skipping\n", - "Layer sha256:9ab35225e174496943b6a86bf62d004409479cf722ef1d3e01ca48afc8cfaa79 exists, skipping\n", - "Layer sha256:c451d20886c33c47dab7b01b05ece292ee5173a9a4aced925035401a6b1de62e exists, skipping\n", - "Layer sha256:afde35469481d2bc446d649a7a3d099147bbf7696b66333e76a411686b617ea1 exists, skipping\n", - "Layer sha256:c5aa14b34cb84e23688babb6c0c39c1591a7dfd7660c719b9d63bdcb8d2aa172 exists, skipping\n", - "Layer sha256:59951887a0c1d1a227f43219b3bc84562a6f2a7e0ab5c276fbd9eaba6ebec02d exists, skipping\n", - "Layer sha256:d76c933128c526784913ade49f5bb957fcdbf340eac405af17aaa01cb9c21650 exists, skipping\n", - "Layer sha256:fa3f2f277e67c5cbbf1dac21dc27111a60d3cd2ef494d94aa1515d3319f2a245 exists, skipping\n", - "Layer sha256:47956fc6abae87d70180bc4f0efdad014b8e2a3b617a447ac01f674336737dfc exists, skipping\n", - "Layer sha256:bd5e67bf2947497b4a4347d2751797d6b3a40f0dc5d355185815ee6da1b8ae0c exists, skipping\n", - "Layer sha256:147c5bbff888fc9cddffd4078daa35bba0d1d6f6c7175a1acb144412a43b3fce exists, skipping\n", - "Layer sha256:86db56dbcdfc4e5ba205e00f3de178548dd0fcd3d1d9ec011747ca0bb08a8177 exists, skipping\n", - "Layer sha256:1d9677f30f0f4a3989f410d16a6829b3953729bb1971eab37e89535308f9bf0a exists, skipping\n", - "Layer sha256:62228d5c51598033083adbf71e8ee3d8d523d7d6d8c9d789b8c8a2d71ca988ac exists, skipping\n", - "Layer sha256:167108358fe643eea57fc595ff9b76a1a7e09e022c84d724346ce5b41d0148bc exists, skipping\n", - "Layer sha256:ff51e784988b3a953df5d6ba36b982436c2b16a77eb081ce7a589ca67d04144c exists, skipping\n", - "Layer sha256:22ea01b3a354ebdcf4386e6d2f53b6cf65bd9cdcb34a70f32e00b90a477589d0 exists, skipping\n", - "Layer sha256:969fc9c5501e60432ca0bc4b635493feb2f90e14822d2f3e3f79742fed96757d exists, skipping\n", - "Layer sha256:398d32b153e84fe343f0c5b07d65e89b05551aae6cb8b3a03bb2b662976eb3b8 exists, skipping\n", - "Layer sha256:547e89bdafacadd9655a394a9d73c49c9890233c0cd244cbc5b1cb859be1395c exists, skipping\n", - "Layer sha256:2db167b6d8ec97ba1ed7e5c1cdb4104ed1a1747d8e19e5d0962a18a08db3577b pushed.\n", - "Layer sha256:d4f27c1c9950aa9a334fdf2a43dddeb0dc51840af8c62d8f4eb7e979023eced9 pushed.\n", - "Finished upload of: gcr.io/jlewi-dev/fairing-job/fairing-job:A84D11BF\n", - "Pushed image gcr.io/jlewi-dev/fairing-job/fairing-job:A84D11BF in 2.6799020239850506s.\n" - ] - } - ], + "outputs": [], "source": [ "preprocessor.preprocess()\n", "\n", @@ -907,7 +558,7 @@ "source": [ "## Launch the K8s Job\n", "\n", - "* You can use fairing to easily launch a [Kubernetes job](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) to invoke code\n", + "* You can use kubeflow fairing to easily launch a [Kubernetes job](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) to invoke code\n", "* You use fairings Kubernetes job library to build a Kubernetes job\n", " * You use pod mutators to attach GCP credentials to the pod\n", " * You can also use pod mutators to attch PVCs\n", @@ -915,148 +566,14 @@ " * In the cell below you extend the command to include `train` as an argument because you want to invoke the train\n", " function\n", " \n", - "**Note** When you invoke train_deployer.deploy; fairing will stream the logs from the Kubernetes job. The job will initially show some connection errors because the job will try to connect to the metadataserver. You can ignore these errors; the job will retry until its able to connect and then continue" + "**Note** When you invoke train_deployer.deploy; kubeflow fairing will stream the logs from the Kubernetes job. The job will initially show some connection errors because the job will try to connect to the metadataserver. You can ignore these errors; the job will retry until its able to connect and then continue" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "The job fairing-job-ftbfh launched.\n", - "Waiting for fairing-job-ftbfh-tm7wg to start...\n", - "Waiting for fairing-job-ftbfh-tm7wg to start...\n", - "Waiting for fairing-job-ftbfh-tm7wg to start...\n", - "Waiting for fairing-job-ftbfh-tm7wg to start...\n", - "Waiting for fairing-job-ftbfh-tm7wg to start...\n", - "Pod started running True\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)': /api/v1alpha1/artifacts\n", - "model_file not supplied; using the default\n", - "model_file=mockup-model.dat\n", - "[0]\tvalidation_0-rmse:93.484\n", - "Will train until validation_0-rmse hasn't improved in 40 rounds.\n", - "[1]\tvalidation_0-rmse:87.8587\n", - "[2]\tvalidation_0-rmse:82.3\n", - "[3]\tvalidation_0-rmse:77.6844\n", - "[4]\tvalidation_0-rmse:73.6202\n", - "[5]\tvalidation_0-rmse:70.051\n", - "[6]\tvalidation_0-rmse:66.3668\n", - "[7]\tvalidation_0-rmse:63.3949\n", - "[8]\tvalidation_0-rmse:60.7882\n", - "[9]\tvalidation_0-rmse:58.6315\n", - "[10]\tvalidation_0-rmse:56.6578\n", - "[11]\tvalidation_0-rmse:54.9536\n", - "[12]\tvalidation_0-rmse:53.4898\n", - "[13]\tvalidation_0-rmse:52.0987\n", - "[14]\tvalidation_0-rmse:50.3687\n", - "[15]\tvalidation_0-rmse:49.4893\n", - "[16]\tvalidation_0-rmse:47.7285\n", - "[17]\tvalidation_0-rmse:46.6625\n", - "[18]\tvalidation_0-rmse:45.5546\n", - "[19]\tvalidation_0-rmse:44.7106\n", - "[20]\tvalidation_0-rmse:43.926\n", - "[21]\tvalidation_0-rmse:43.1622\n", - "[22]\tvalidation_0-rmse:42.2623\n", - "[23]\tvalidation_0-rmse:41.4546\n", - "[24]\tvalidation_0-rmse:40.7816\n", - "[25]\tvalidation_0-rmse:39.9577\n", - "[26]\tvalidation_0-rmse:39.303\n", - "[27]\tvalidation_0-rmse:38.7521\n", - "[28]\tvalidation_0-rmse:38.1119\n", - "[29]\tvalidation_0-rmse:37.3337\n", - "[30]\tvalidation_0-rmse:37.2111\n", - "[31]\tvalidation_0-rmse:36.7978\n", - "[32]\tvalidation_0-rmse:36.0016\n", - "[33]\tvalidation_0-rmse:35.5342\n", - "[34]\tvalidation_0-rmse:35.3714\n", - "[35]\tvalidation_0-rmse:34.7494\n", - "[36]\tvalidation_0-rmse:34.5603\n", - "[37]\tvalidation_0-rmse:34.126\n", - "[38]\tvalidation_0-rmse:33.8422\n", - "[39]\tvalidation_0-rmse:33.6291\n", - "[40]\tvalidation_0-rmse:33.2366\n", - "[41]\tvalidation_0-rmse:32.8506\n", - "[42]\tvalidation_0-rmse:32.7166\n", - "[43]\tvalidation_0-rmse:32.6814\n", - "[44]\tvalidation_0-rmse:32.3616\n", - "[45]\tvalidation_0-rmse:32.1414\n", - "[46]\tvalidation_0-rmse:32.0641\n", - "[47]\tvalidation_0-rmse:31.6858\n", - "[48]\tvalidation_0-rmse:31.6368\n", - "[49]\tvalidation_0-rmse:31.3603\n", - "mean_absolute_error=22.90\n", - "Model export success: mockup-model.dat\n", - "Best RMSE on eval: %.2f with %d rounds 31.360287 50\n" - ] - } - ], + "outputs": [], "source": [ "pod_spec = builder.generate_pod_spec()\n", "train_deployer = job.job.Job(cleanup=False,\n", @@ -1077,88 +594,9 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "apiVersion: v1\n", - "items:\n", - "- apiVersion: batch/v1\n", - " kind: Job\n", - " metadata:\n", - " creationTimestamp: \"2019-08-18T00:33:14Z\"\n", - " generateName: fairing-job-\n", - " labels:\n", - " fairing-deployer: job\n", - " fairing-id: c423b296-c14f-11e9-8c24-dade65de3221\n", - " name: fairing-job-ftbfh\n", - " namespace: kubeflow-jlewi\n", - " resourceVersion: \"851927\"\n", - " selfLink: /apis/batch/v1/namespaces/kubeflow-jlewi/jobs/fairing-job-ftbfh\n", - " uid: c42a1de3-c14f-11e9-ab37-42010a8e00c9\n", - " spec:\n", - " backoffLimit: 0\n", - " completions: 1\n", - " parallelism: 1\n", - " selector:\n", - " matchLabels:\n", - " controller-uid: c42a1de3-c14f-11e9-ab37-42010a8e00c9\n", - " template:\n", - " metadata:\n", - " creationTimestamp: null\n", - " labels:\n", - " controller-uid: c42a1de3-c14f-11e9-ab37-42010a8e00c9\n", - " fairing-deployer: job\n", - " fairing-id: c423b296-c14f-11e9-8c24-dade65de3221\n", - " job-name: fairing-job-ftbfh\n", - " name: fairing-deployer\n", - " spec:\n", - " containers:\n", - " - command:\n", - " - python\n", - " - /app/build-train-deploy.py\n", - " - train\n", - " env:\n", - " - name: FAIRING_RUNTIME\n", - " value: \"1\"\n", - " - name: GOOGLE_APPLICATION_CREDENTIALS\n", - " value: /etc/secrets/user-gcp-sa.json\n", - " image: gcr.io/jlewi-dev/fairing-job/fairing-job:A84D11BF\n", - " imagePullPolicy: IfNotPresent\n", - " name: fairing-job\n", - " resources: {}\n", - " securityContext:\n", - " runAsUser: 0\n", - " terminationMessagePath: /dev/termination-log\n", - " terminationMessagePolicy: File\n", - " volumeMounts:\n", - " - mountPath: /etc/secrets\n", - " name: user-gcp-sa\n", - " readOnly: true\n", - " workingDir: /app/\n", - " dnsPolicy: ClusterFirst\n", - " restartPolicy: Never\n", - " schedulerName: default-scheduler\n", - " securityContext: {}\n", - " terminationGracePeriodSeconds: 30\n", - " volumes:\n", - " - name: user-gcp-sa\n", - " secret:\n", - " defaultMode: 420\n", - " secretName: user-gcp-sa\n", - " status:\n", - " active: 1\n", - " startTime: \"2019-08-18T00:33:14Z\"\n", - "kind: List\n", - "metadata:\n", - " resourceVersion: \"\"\n", - " selfLink: \"\"\n" - ] - } - ], + "outputs": [], "source": [ "!kubectl get jobs -l fairing-id={train_deployer.job_id} -o yaml" ] @@ -1174,28 +612,20 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "* Now that you have trained a model you can use fairing to deploy it on Kubernetes\n", + "* Now that you have trained a model you can use kubeflow fairing to deploy it on Kubernetes\n", "* When you call deployer.deploy fairing will create a [Kubernetes Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) to serve your model\n", - "* Fairing uses the docker image you created earlier\n", + "* Kubeflow fairing uses the docker image you created earlier\n", "* The docker image you created contains your code and [Seldon core](https://www.seldon.io/)\n", - "* Fairing uses Seldon to wrap your prediction code, ModelServe.predict, in a REST and gRPC server" + "* Kubeflow fairing uses Seldon to wrap your prediction code, ModelServe.predict, in a REST and gRPC server" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Cluster endpoint: http://fairing-service-cxwv5.kubeflow-jlewi.svc.cluster.local\n" - ] - } - ], + "outputs": [], "source": [ - "from fairing.deployers import serving\n", + "from kubeflow.fairing.deployers import serving\n", "pod_spec = builder.generate_pod_spec()\n", "\n", "module_name = os.path.splitext(preprocessor.executable.name)[0]\n", @@ -1215,98 +645,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "apiVersion: extensions/v1beta1\n", - "kind: Deployment\n", - "metadata:\n", - " annotations:\n", - " deployment.kubernetes.io/revision: \"1\"\n", - " creationTimestamp: \"2019-08-18T00:33:33Z\"\n", - " generateName: fairing-deployer-\n", - " generation: 1\n", - " labels:\n", - " app: mockup\n", - " fairing-deployer: serving\n", - " fairing-id: cf47840e-c14f-11e9-8c24-dade65de3221\n", - " name: fairing-deployer-lcwvh\n", - " namespace: kubeflow-jlewi\n", - " resourceVersion: \"852066\"\n", - " selfLink: /apis/extensions/v1beta1/namespaces/kubeflow-jlewi/deployments/fairing-deployer-lcwvh\n", - " uid: cf49d3a5-c14f-11e9-ab37-42010a8e00c9\n", - "spec:\n", - " progressDeadlineSeconds: 600\n", - " replicas: 1\n", - " revisionHistoryLimit: 10\n", - " selector:\n", - " matchLabels:\n", - " app: mockup\n", - " fairing-deployer: serving\n", - " fairing-id: cf47840e-c14f-11e9-8c24-dade65de3221\n", - " strategy:\n", - " rollingUpdate:\n", - " maxSurge: 25%\n", - " maxUnavailable: 25%\n", - " type: RollingUpdate\n", - " template:\n", - " metadata:\n", - " creationTimestamp: null\n", - " labels:\n", - " app: mockup\n", - " fairing-deployer: serving\n", - " fairing-id: cf47840e-c14f-11e9-8c24-dade65de3221\n", - " name: fairing-deployer\n", - " spec:\n", - " containers:\n", - " - command:\n", - " - seldon-core-microservice\n", - " - build-train-deploy.ModelServe\n", - " - REST\n", - " - --service-type=MODEL\n", - " - --persistence=0\n", - " env:\n", - " - name: FAIRING_RUNTIME\n", - " value: \"1\"\n", - " image: gcr.io/jlewi-dev/fairing-job/fairing-job:A84D11BF\n", - " imagePullPolicy: IfNotPresent\n", - " name: model\n", - " resources: {}\n", - " securityContext:\n", - " runAsUser: 0\n", - " terminationMessagePath: /dev/termination-log\n", - " terminationMessagePolicy: File\n", - " workingDir: /app/\n", - " dnsPolicy: ClusterFirst\n", - " restartPolicy: Always\n", - " schedulerName: default-scheduler\n", - " securityContext: {}\n", - " terminationGracePeriodSeconds: 30\n", - "status:\n", - " conditions:\n", - " - lastTransitionTime: \"2019-08-18T00:33:33Z\"\n", - " lastUpdateTime: \"2019-08-18T00:33:33Z\"\n", - " message: Deployment does not have minimum availability.\n", - " reason: MinimumReplicasUnavailable\n", - " status: \"False\"\n", - " type: Available\n", - " - lastTransitionTime: \"2019-08-18T00:33:33Z\"\n", - " lastUpdateTime: \"2019-08-18T00:33:33Z\"\n", - " message: ReplicaSet \"fairing-deployer-lcwvh-64bbfd968c\" is progressing.\n", - " reason: ReplicaSetUpdated\n", - " status: \"True\"\n", - " type: Progressing\n", - " observedGeneration: 1\n", - " replicas: 1\n", - " unavailableReplicas: 1\n", - " updatedReplicas: 1\n" - ] - } - ], + "outputs": [], "source": [ "!kubectl get deploy -o yaml {deployer.deployment.metadata.name}" ] @@ -1324,7 +665,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1333,18 +674,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(b'{\"data\":{\"names\":[\"t:0\",\"t:1\"],\"tensor\":{\"shape\":[1,2],\"values\":[-27.2208251'\n", - " b'953125,-36.85965347290039]}},\"meta\":{}}\\n')\n" - ] - } - ], + "outputs": [], "source": [ "full_url = url + \":5000/predict\"\n", "result = util.predict_nparray(full_url, test_X)\n", @@ -1363,7 +695,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1398,383 +730,9 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'id': '4',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T17:37:21.338937',\n", - " 'create_time': '2019-08-16T17:37:21.569390Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T17:37:21.569356',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '7',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T17:37:40.551731',\n", - " 'create_time': '2019-08-16T17:37:40.762260Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T17:37:40.762231',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '10',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T17:37:42.075423',\n", - " 'create_time': '2019-08-16T17:37:42.288795Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T17:37:42.288767',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '13',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T17:37:43.596662',\n", - " 'create_time': '2019-08-16T17:37:43.797385Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T17:37:43.797357',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '16',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T17:37:45.697220',\n", - " 'create_time': '2019-08-16T17:37:45.902046Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T17:37:45.901973',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '19',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T22:10:53.508578',\n", - " 'create_time': '2019-08-16T22:10:53.761012Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T22:10:53.760970',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '22',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T22:10:56.902050',\n", - " 'create_time': '2019-08-16T22:10:57.126435Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T22:10:57.126410',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '25',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T22:34:04.140210',\n", - " 'create_time': '2019-08-16T22:34:04.371380Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T22:34:04.371342',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '28',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T22:35:42.154942',\n", - " 'create_time': '2019-08-16T22:35:42.400014Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T22:35:42.399985',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '31',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T22:50:00.175507',\n", - " 'create_time': '2019-08-16T22:50:00.398220Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T22:50:00.398189',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '34',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T22:50:09.199737',\n", - " 'create_time': '2019-08-16T22:50:09.409746Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T22:50:09.409716',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '37',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T22:55:11.185839',\n", - " 'create_time': '2019-08-16T22:55:11.426015Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T22:55:11.425985',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '40',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T22:56:58.371363',\n", - " 'create_time': '2019-08-16T22:56:58.545544Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T22:56:58.545506',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '43',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T23:35:08.335115',\n", - " 'create_time': '2019-08-16T23:35:08.520758Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T23:35:08.520583',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '46',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T23:44:46.889414',\n", - " 'create_time': '2019-08-16T23:44:47.131904Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T23:44:47.131868',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '49',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T23:51:34.813950',\n", - " 'create_time': '2019-08-16T23:51:35.084604Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T23:51:35.084556',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '52',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-16T23:57:01.558106',\n", - " 'create_time': '2019-08-16T23:57:01.750162Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-16T23:57:01.750132',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '55',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-17T00:02:40.138471',\n", - " 'create_time': '2019-08-17T00:02:40.409795Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-17T00:02:40.409763',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '58',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-17T00:04:02.248451',\n", - " 'create_time': '2019-08-17T00:04:02.407539Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-17T00:04:02.407509',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '61',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-17T01:13:43.418039',\n", - " 'create_time': '2019-08-17T01:13:43.676249Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-17T01:13:43.676202',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '64',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-17T01:48:09.879955',\n", - " 'create_time': '2019-08-17T01:48:10.090521Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-17T01:48:10.090478',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '67',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-17T20:14:31.424733',\n", - " 'create_time': '2019-08-17T20:14:31.624324Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-17T20:14:31.624290',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '70',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-17T21:31:07.792382',\n", - " 'create_time': '2019-08-17T21:31:07.979491Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-17T21:31:07.979460',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '73',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-17T21:32:19.864465',\n", - " 'create_time': '2019-08-17T21:32:20.051897Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-17T21:32:20.051859',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '76',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-17T21:34:13.957359',\n", - " 'create_time': '2019-08-17T21:34:14.154483Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-17T21:34:14.154453',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '79',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-17T21:34:40.198663',\n", - " 'create_time': '2019-08-17T21:34:40.386833Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-17T21:34:40.386797',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '82',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-18T00:32:40.451885',\n", - " 'create_time': '2019-08-18T00:32:40.678094Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-18T00:32:40.678065',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None},\n", - " {'id': '85',\n", - " 'workspace': 'xgboost-synthetic',\n", - " 'run': 'xgboost-synthetic-faring-run2019-08-18T00:33:27.326993',\n", - " 'create_time': '2019-08-18T00:33:27.509801Z',\n", - " 'description': 'housing price prediction model using synthetic data',\n", - " 'model_type': 'linear_regression',\n", - " 'name': 'housing-price-model',\n", - " 'owner': 'someone@kubeflow.org',\n", - " 'version': '2019-08-18T00:33:27.509766',\n", - " 'uri': 'mockup-model.dat',\n", - " 'training_framework': {'name': 'xgboost', 'version': '0.9.0'},\n", - " 'hyperparameters': {'learning_rate': 0.1, 'n_estimators': 50},\n", - " 'labels': None}]" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ws = create_workspace()\n", "ws.list()" @@ -1804,14 +762,14 @@ "* Inside the function, each step in the function is defined by a ContainerOp that specifies\n", " a container to invoke\n", " \n", - "* You will use the container image that you built earlier using Fairing\n", - "* Since the Fairing preprocessor added a main function using [python-fire](https://github.com/google/python-fire), a step in your pipeline can invocation any function in the ModelServe class just by setting the command for the container op\n", + "* You will use the container image that you built earlier using Kubeflow Fairing\n", + "* Since the Kubeflow Fairing preprocessor added a main function using [python-fire](https://github.com/google/python-fire), a step in your pipeline can invocation any function in the ModelServe class just by setting the command for the container op\n", "* See the pipelines [SDK reference](https://kubeflow-pipelines.readthedocs.io/en/latest/) for more information" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1848,7 +806,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1875,34 +833,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "Experiment link here" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Run link here" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "EXPERIMENT_NAME = 'MockupModel'\n", "\n", diff --git a/xgboost_synthetic/requirements.txt b/xgboost_synthetic/requirements.txt index 1b78fca8..32821f16 100644 --- a/xgboost_synthetic/requirements.txt +++ b/xgboost_synthetic/requirements.txt @@ -1,4 +1,3 @@ -fairing fire gitpython google-cloud-storage