mirror of https://github.com/kubeflow/examples.git
Install pip dependencies and build base image with kaniko (#603)
* Install required pip packages not included in the base package. * Use Kaniko builder to build the base image first. * Directly install packages from requirements.txt to be more flexible.
This commit is contained in:
Chun-Hsiang Wang
Kubernetes Prow Robot
parent
cda6efed27
commit
fb6cd69def
|
|
@ -32,7 +32,9 @@
|
|||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"metadata": {
|
||||
"scrolled": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import util\n",
|
||||
|
|
@ -43,7 +45,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -68,7 +70,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -87,7 +89,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -110,7 +112,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -161,7 +163,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -217,7 +219,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -225,72 +227,73 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
"model_file=mockup-model.dat\n",
|
||||
"[0]\tvalidation_0-rmse:97.625\n",
|
||||
"[14:45:28] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
|
||||
"[0]\tvalidation_0-rmse:95.4029\n",
|
||||
"Will train until validation_0-rmse hasn't improved in 40 rounds.\n",
|
||||
"[1]\tvalidation_0-rmse:92.9346\n",
|
||||
"[2]\tvalidation_0-rmse:88.4163\n",
|
||||
"[3]\tvalidation_0-rmse:84.9513\n",
|
||||
"[4]\tvalidation_0-rmse:81.4807\n",
|
||||
"[5]\tvalidation_0-rmse:78.0301\n",
|
||||
"[6]\tvalidation_0-rmse:74.3916\n",
|
||||
"[7]\tvalidation_0-rmse:72.6324\n",
|
||||
"[8]\tvalidation_0-rmse:70.0073\n",
|
||||
"[9]\tvalidation_0-rmse:67.4423\n",
|
||||
"[10]\tvalidation_0-rmse:66.0759\n",
|
||||
"[11]\tvalidation_0-rmse:63.7281\n",
|
||||
"[12]\tvalidation_0-rmse:61.7721\n",
|
||||
"[13]\tvalidation_0-rmse:59.8362\n",
|
||||
"[14]\tvalidation_0-rmse:58.0936\n",
|
||||
"[15]\tvalidation_0-rmse:56.2871\n",
|
||||
"[16]\tvalidation_0-rmse:54.6282\n",
|
||||
"[17]\tvalidation_0-rmse:53.242\n",
|
||||
"[18]\tvalidation_0-rmse:51.9367\n",
|
||||
"[19]\tvalidation_0-rmse:50.4069\n",
|
||||
"[20]\tvalidation_0-rmse:49.4686\n",
|
||||
"[21]\tvalidation_0-rmse:48.2332\n",
|
||||
"[22]\tvalidation_0-rmse:47.4084\n",
|
||||
"[23]\tvalidation_0-rmse:46.8214\n",
|
||||
"[24]\tvalidation_0-rmse:46.1743\n",
|
||||
"[25]\tvalidation_0-rmse:45.2428\n",
|
||||
"[26]\tvalidation_0-rmse:44.6314\n",
|
||||
"[27]\tvalidation_0-rmse:43.7469\n",
|
||||
"[28]\tvalidation_0-rmse:42.8601\n",
|
||||
"[29]\tvalidation_0-rmse:41.9884\n",
|
||||
"[30]\tvalidation_0-rmse:41.384\n",
|
||||
"[31]\tvalidation_0-rmse:40.8639\n",
|
||||
"[32]\tvalidation_0-rmse:40.1512\n",
|
||||
"[33]\tvalidation_0-rmse:39.5409\n",
|
||||
"[34]\tvalidation_0-rmse:39.0861\n",
|
||||
"[35]\tvalidation_0-rmse:38.3517\n",
|
||||
"[36]\tvalidation_0-rmse:37.8571\n",
|
||||
"[37]\tvalidation_0-rmse:37.5808\n",
|
||||
"[38]\tvalidation_0-rmse:36.9849\n",
|
||||
"[39]\tvalidation_0-rmse:36.5718\n",
|
||||
"[40]\tvalidation_0-rmse:36.1384\n",
|
||||
"[41]\tvalidation_0-rmse:35.7462\n",
|
||||
"[42]\tvalidation_0-rmse:35.2703\n",
|
||||
"[43]\tvalidation_0-rmse:34.8709\n",
|
||||
"[44]\tvalidation_0-rmse:34.4978\n",
|
||||
"[45]\tvalidation_0-rmse:34.1141\n",
|
||||
"[46]\tvalidation_0-rmse:33.7975\n",
|
||||
"[47]\tvalidation_0-rmse:33.4405\n",
|
||||
"[48]\tvalidation_0-rmse:33.0749\n",
|
||||
"[49]\tvalidation_0-rmse:32.7983\n"
|
||||
"[1]\tvalidation_0-rmse:88.2563\n",
|
||||
"[2]\tvalidation_0-rmse:82.341\n",
|
||||
"[3]\tvalidation_0-rmse:76.7723\n",
|
||||
"[4]\tvalidation_0-rmse:71.9907\n",
|
||||
"[5]\tvalidation_0-rmse:67.7698\n",
|
||||
"[6]\tvalidation_0-rmse:63.2959\n",
|
||||
"[7]\tvalidation_0-rmse:59.8439\n",
|
||||
"[8]\tvalidation_0-rmse:56.9911\n",
|
||||
"[9]\tvalidation_0-rmse:53.8091\n",
|
||||
"[10]\tvalidation_0-rmse:51.4086\n",
|
||||
"[11]\tvalidation_0-rmse:49.1506\n",
|
||||
"[12]\tvalidation_0-rmse:47.4958\n",
|
||||
"[13]\tvalidation_0-rmse:46.0659\n",
|
||||
"[14]\tvalidation_0-rmse:44.4425\n",
|
||||
"[15]\tvalidation_0-rmse:42.5323\n",
|
||||
"[16]\tvalidation_0-rmse:41.3847\n",
|
||||
"[17]\tvalidation_0-rmse:40.4265\n",
|
||||
"[18]\tvalidation_0-rmse:39.2736\n",
|
||||
"[19]\tvalidation_0-rmse:38.5218\n",
|
||||
"[20]\tvalidation_0-rmse:37.3415\n",
|
||||
"[21]\tvalidation_0-rmse:36.8546\n",
|
||||
"[22]\tvalidation_0-rmse:36.0049\n",
|
||||
"[23]\tvalidation_0-rmse:35.5978\n",
|
||||
"[24]\tvalidation_0-rmse:35.0653\n",
|
||||
"[25]\tvalidation_0-rmse:34.1586\n",
|
||||
"[26]\tvalidation_0-rmse:33.6017\n",
|
||||
"[27]\tvalidation_0-rmse:33.2441\n",
|
||||
"[28]\tvalidation_0-rmse:32.477\n",
|
||||
"[29]\tvalidation_0-rmse:31.7638\n",
|
||||
"[30]\tvalidation_0-rmse:31.2781\n",
|
||||
"[31]\tvalidation_0-rmse:30.9532\n",
|
||||
"[32]\tvalidation_0-rmse:30.3881\n",
|
||||
"[33]\tvalidation_0-rmse:29.9289\n",
|
||||
"[34]\tvalidation_0-rmse:29.6362\n",
|
||||
"[35]\tvalidation_0-rmse:29.3138\n",
|
||||
"[36]\tvalidation_0-rmse:29.0621\n",
|
||||
"[37]\tvalidation_0-rmse:28.5649\n",
|
||||
"[38]\tvalidation_0-rmse:28.15\n",
|
||||
"[39]\tvalidation_0-rmse:27.8467\n",
|
||||
"[40]\tvalidation_0-rmse:27.5816\n",
|
||||
"[41]\tvalidation_0-rmse:27.4534\n",
|
||||
"[42]\tvalidation_0-rmse:27.2668\n",
|
||||
"[43]\tvalidation_0-rmse:27.0583\n",
|
||||
"[44]\tvalidation_0-rmse:26.7226\n",
|
||||
"[45]\tvalidation_0-rmse:26.6145\n",
|
||||
"[46]\tvalidation_0-rmse:26.3878\n",
|
||||
"[47]\tvalidation_0-rmse:26.029\n",
|
||||
"[48]\tvalidation_0-rmse:25.8776\n",
|
||||
"[49]\tvalidation_0-rmse:25.8484\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"INFO:root:mean_absolute_error=25.64\n",
|
||||
"INFO:root:Model export success: mockup-model.dat\n"
|
||||
"mean_absolute_error=19.92\n",
|
||||
"Model export success: mockup-model.dat\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Best RMSE on eval: %.2f with %d rounds 32.798336 50\n"
|
||||
"Best RMSE on eval: %.2f with %d rounds 25.848402 50\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -309,7 +312,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -317,16 +320,17 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
"model_file not supplied; using the default\n",
|
||||
"model_file=mockup-model.dat\n"
|
||||
"model_file=mockup-model.dat\n",
|
||||
"[14:45:28] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[[-37.04857635498047, -37.04857635498047]]"
|
||||
"[[68.33491516113281, 68.33491516113281]]"
|
||||
]
|
||||
},
|
||||
"execution_count": 16,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
|
@ -355,15 +359,15 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"zahrakubeflowcodelab\n",
|
||||
"gcr.io/zahrakubeflowcodelab/fairing-job\n"
|
||||
"issue-label-bot-dev\n",
|
||||
"gcr.io/issue-label-bot-dev/fairing-job\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -391,7 +395,126 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[PosixPath('build-train-deploy.py'), 'xgboost_util.py', 'mockup-model.dat']"
|
||||
]
|
||||
},
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from fairing.builders import cluster\n",
|
||||
"preprocessor = ConvertNotebookPreprocessorWithFire(\"HousingServe\")\n",
|
||||
"\n",
|
||||
"if not preprocessor.input_files:\n",
|
||||
" preprocessor.input_files = set()\n",
|
||||
"input_files=[\"xgboost_util.py\", \"mockup-model.dat\"]\n",
|
||||
"preprocessor.input_files = set([os.path.normpath(f) for f in input_files])\n",
|
||||
"preprocessor.preprocess()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Building image using cluster builder.\n",
|
||||
"Creating docker context: /tmp/fairing_context_5d629kor\n",
|
||||
"Waiting for fairing-builder-lz9zx to start...\n",
|
||||
"Pod started running True\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[36mINFO\u001b[0m[0000] Downloading base image gcr.io/kubeflow-images-public/xgboost-fairing-example-base:v-20190612\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Downloading base image gcr.io/kubeflow-images-public/xgboost-fairing-example-base:v-20190612\n",
|
||||
"\u001b[33mWARN\u001b[0m[0000] Error while retrieving image from cache: getting image from path: open /cache/sha256:f90e54e312c4cfba28bec6993add2a85b4e127b52149ec0aaf41e5f8889a4086: no such file or directory\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Checking for cached layer gcr.io/issue-label-bot-dev/fairing-job/fairing-job/cache:e46cfa04f5f0d0445ce3ce8b91886d94e96f2875510a69aa9afaeb0ba9e62fc4...\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Using caching version of cmd: RUN if [ -e requirements.txt ];then pip install --no-cache -r requirements.txt; fi\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Using files from context: [/kaniko/buildcontext/app]\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Taking snapshot of full filesystem...\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Skipping paths under /dev, as it is a whitelisted directory\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Skipping paths under /etc/secrets, as it is a whitelisted directory\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Skipping paths under /kaniko, as it is a whitelisted directory\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Skipping paths under /proc, as it is a whitelisted directory\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Skipping paths under /sys, as it is a whitelisted directory\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Skipping paths under /var/run, as it is a whitelisted directory\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] WORKDIR /app/\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] cmd: workdir\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Changed working directory to /app/\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Creating directory /app/\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Taking snapshot of files...\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] ENV FAIRING_RUNTIME 1\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] No files changed in this command, skipping snapshotting.\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] RUN if [ -e requirements.txt ];then pip install --no-cache -r requirements.txt; fi\n",
|
||||
"\u001b[36mINFO\u001b[0m[0000] Found cached layer, extracting to filesystem\n",
|
||||
"\u001b[36mINFO\u001b[0m[0001] No files changed in this command, skipping snapshotting.\n",
|
||||
"\u001b[36mINFO\u001b[0m[0001] Using files from context: [/kaniko/buildcontext/app]\n",
|
||||
"\u001b[36mINFO\u001b[0m[0001] COPY /app/ /app/\n",
|
||||
"\u001b[36mINFO\u001b[0m[0001] Taking snapshot of files...\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:d13453f7d2b8d0adfd86c3989a5b695cef5afc3efaafe559643071f258c9f06d\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:0ba512db704a2eb85f7f372d1c809d58589531e3bae794f0aaba86cee912f923\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:9ee379bde91a3cecfb08d4189af0a2bcecc2da1c5102e49443088ccd7bd9abfa\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:507170ae8cfaca6cf2999295221d1324f1051fa15ba59e04dd7dafdc8de565bc\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:2f1ee468081da0ca09360c50281ed261d8b3fb01f664262c3f278d8619eb4e9a\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:d099b15c53311dc296426716edabe61dcc19e88009c19098b17ba965357c4391\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:bad6918fba4b1c68f82d1a4b6063b3ce64975a73b33b38b35454b1d484a6b57b\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:0fd02182c40eb28e13c4d7efd5dd4c81d985d9b07c9c809cc26e7bdb2dced07e\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:079dd3e30fa3eed702bb20a2f725da9907e2732bdc4dfb2fb5084a3423c3f743\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:e7fea64fabbc6f5961864ce5c6bcc143ab616d325b0c5a26848d8e427806104f\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:a5ba9de0ac70b35658f5898c27b52063a597d790308fb853021e881e04a6efb7\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:124c757242f88002a858c23fc79f8262f9587fa30fd92507e586ad074afb42b6\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:bbf0f5f91e8108d9b0be1ceeb749e63788ce7394a184bc8a70d24017eca7b7ba\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:9d866f8bde2a0d607a6d17edc0fbd5e00b58306efc2b0a57e0ba72f269e7c6be\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:afde35469481d2bc446d649a7a3d099147bbf7696b66333e76a411686b617ea1\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:398d32b153e84fe343f0c5b07d65e89b05551aae6cb8b3a03bb2b662976eb3b8\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:55dbf73eb7c7c005c3ccff29b62ff180e2f29245d14794dd6d5d8ad855d0ea88\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:4bfa6a63a3897359eff3ca3ee27c2e05ba76b790a07e6583714c1d324c2d4f21\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:5d8a6f34a39a1e098f09b39ee4e9d4a178fef6ec71c2046fe0b040c4667c8143\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:b893ca5fa31bb87be0d3fa3a403dac7ca12c955d6fd522fd35e3260dbd0e99da\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:ecc17173ccb5b7692a6d31b0077b8e4f543fb45f8c2b5c252dcad9ad0c9be0f7\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:eed14867f5ee443ad7efc89d0d4392683799a413244feec120f43074bc2d43ef\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:07e06c833ecb3b115e378d7f2ba5817ba77cfd02f5794a9817ede0622fbbf8a5\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:541a15d3a9d79f7d3e5e0f552f396406b3e3093247f71e0ae71dd8b7242ec428\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:fa3f2f277e67c5cbbf1dac21dc27111a60d3cd2ef494d94aa1515d3319f2a245\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:8143617e89d7ba1957e3dc6d7093a48bd0cd4a2a709bc0c9d0ffc6dde11467e8\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:2327f2e2474891211dbf7fb2d54e16e7b2889fea157b726645cc05e75ad917e8\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:8c58e650bb886ab24426958165c15abe1a1c10e8710f50233701fd503e23e7ac\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:90a7e2cb4d7460e55f83c6e47f9f8d089895ee6e1cc51ae5c23eab3bdcb70363\n",
|
||||
"2019/07/18 21:45:45 existing blob: sha256:1cf84c00b8903926c231b4b5974c0419556a4a578bf9416f585fcbf1b7aa70ab\n",
|
||||
"2019/07/18 21:45:46 pushed blob sha256:8ab941f264e893bf2d02a0f6d2972fa5f725995cba85b0a897cee1531525bba1\n",
|
||||
"2019/07/18 21:45:46 pushed blob sha256:acb611ba3316584866914521fe68dd9892e3fea865900f7c15f2f7268587cd93\n",
|
||||
"2019/07/18 21:45:46 pushed blob sha256:80794aeb9ef80da69469ae895f20899b52d9115e4161543c83774863e97fc507\n",
|
||||
"2019/07/18 21:45:47 gcr.io/issue-label-bot-dev/fairing-job/fairing-job:E480ACAF: digest: sha256:1c10c3629d920b78e54f16fe268eb77f976d1ff5a48b31a9f54df478ff012a2a size: 5468\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"cluster_builder = cluster.cluster.ClusterBuilder(registry=DOCKER_REGISTRY,\n",
|
||||
" base_image=base_image,\n",
|
||||
" namespace='kubeflow',\n",
|
||||
" preprocessor=preprocessor,\n",
|
||||
" pod_spec_mutators=[fairing.cloud.gcp.add_gcp_credentials_if_exists],\n",
|
||||
" context_source=cluster.gcs_context.GCSContextSource())\n",
|
||||
"cluster_builder.build()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
|
|
@ -400,64 +523,60 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"WARNING:fairing.builders.append.append:Building image using Append builder...\n",
|
||||
"INFO:root:Creating docker context: /tmp/fairing_context_de6bgft2\n",
|
||||
"INFO:root:Loading Docker credentials for repository 'gcr.io/kubeflow-images-public/xgboost-fairing-example-base:v-20190612'\n",
|
||||
"INFO:root:Invoking 'docker-credential-gcloud' to obtain Docker credentials.\n",
|
||||
"INFO:root:Successfully obtained Docker credentials.\n",
|
||||
"WARNING:fairing.builders.append.append:Image successfully built in 1.157013630028814s.\n",
|
||||
"WARNING:fairing.builders.append.append:Pushing image gcr.io/zahrakubeflowcodelab/fairing-job/fairing-job:6F63F28C...\n",
|
||||
"INFO:root:Loading Docker credentials for repository 'gcr.io/zahrakubeflowcodelab/fairing-job/fairing-job:6F63F28C'\n",
|
||||
"INFO:root:Invoking 'docker-credential-gcloud' to obtain Docker credentials.\n",
|
||||
"INFO:root:Successfully obtained Docker credentials.\n",
|
||||
"WARNING:fairing.builders.append.append:Uploading gcr.io/zahrakubeflowcodelab/fairing-job/fairing-job:6F63F28C\n",
|
||||
"INFO:root:Layer sha256:2f1ee468081da0ca09360c50281ed261d8b3fb01f664262c3f278d8619eb4e9a exists, skipping\n",
|
||||
"INFO:root:Layer sha256:90a7e2cb4d7460e55f83c6e47f9f8d089895ee6e1cc51ae5c23eab3bdcb70363 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:b893ca5fa31bb87be0d3fa3a403dac7ca12c955d6fd522fd35e3260dbd0e99da exists, skipping\n",
|
||||
"INFO:root:Layer sha256:eed14867f5ee443ad7efc89d0d4392683799a413244feec120f43074bc2d43ef exists, skipping\n",
|
||||
"INFO:root:Layer sha256:afde35469481d2bc446d649a7a3d099147bbf7696b66333e76a411686b617ea1 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:124c757242f88002a858c23fc79f8262f9587fa30fd92507e586ad074afb42b6 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:fa3f2f277e67c5cbbf1dac21dc27111a60d3cd2ef494d94aa1515d3319f2a245 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:07e06c833ecb3b115e378d7f2ba5817ba77cfd02f5794a9817ede0622fbbf8a5 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:507170ae8cfaca6cf2999295221d1324f1051fa15ba59e04dd7dafdc8de565bc exists, skipping\n",
|
||||
"INFO:root:Layer sha256:0ba512db704a2eb85f7f372d1c809d58589531e3bae794f0aaba86cee912f923 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:079dd3e30fa3eed702bb20a2f725da9907e2732bdc4dfb2fb5084a3423c3f743 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:a5ba9de0ac70b35658f5898c27b52063a597d790308fb853021e881e04a6efb7 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:541a15d3a9d79f7d3e5e0f552f396406b3e3093247f71e0ae71dd8b7242ec428 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:9d866f8bde2a0d607a6d17edc0fbd5e00b58306efc2b0a57e0ba72f269e7c6be exists, skipping\n",
|
||||
"INFO:root:Layer sha256:8143617e89d7ba1957e3dc6d7093a48bd0cd4a2a709bc0c9d0ffc6dde11467e8 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:398d32b153e84fe343f0c5b07d65e89b05551aae6cb8b3a03bb2b662976eb3b8 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:d13453f7d2b8d0adfd86c3989a5b695cef5afc3efaafe559643071f258c9f06d exists, skipping\n",
|
||||
"INFO:root:Layer sha256:9ee379bde91a3cecfb08d4189af0a2bcecc2da1c5102e49443088ccd7bd9abfa exists, skipping\n",
|
||||
"INFO:root:Layer sha256:2327f2e2474891211dbf7fb2d54e16e7b2889fea157b726645cc05e75ad917e8 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:0fd02182c40eb28e13c4d7efd5dd4c81d985d9b07c9c809cc26e7bdb2dced07e exists, skipping\n",
|
||||
"INFO:root:Layer sha256:d099b15c53311dc296426716edabe61dcc19e88009c19098b17ba965357c4391 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:ecc17173ccb5b7692a6d31b0077b8e4f543fb45f8c2b5c252dcad9ad0c9be0f7 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:4bfa6a63a3897359eff3ca3ee27c2e05ba76b790a07e6583714c1d324c2d4f21 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:bbf0f5f91e8108d9b0be1ceeb749e63788ce7394a184bc8a70d24017eca7b7ba exists, skipping\n",
|
||||
"INFO:root:Layer sha256:1cf84c00b8903926c231b4b5974c0419556a4a578bf9416f585fcbf1b7aa70ab exists, skipping\n",
|
||||
"INFO:root:Layer sha256:e7fea64fabbc6f5961864ce5c6bcc143ab616d325b0c5a26848d8e427806104f exists, skipping\n",
|
||||
"INFO:root:Layer sha256:55dbf73eb7c7c005c3ccff29b62ff180e2f29245d14794dd6d5d8ad855d0ea88 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:8c58e650bb886ab24426958165c15abe1a1c10e8710f50233701fd503e23e7ac exists, skipping\n",
|
||||
"INFO:root:Layer sha256:bad6918fba4b1c68f82d1a4b6063b3ce64975a73b33b38b35454b1d484a6b57b exists, skipping\n",
|
||||
"INFO:root:Layer sha256:5d8a6f34a39a1e098f09b39ee4e9d4a178fef6ec71c2046fe0b040c4667c8143 exists, skipping\n",
|
||||
"INFO:root:Layer sha256:da6ff2e3a7d15c18b14d5ea8dbb333b0c912de8d39cee59de1a60bc3926ea0c2 pushed.\n",
|
||||
"INFO:root:Layer sha256:4014b6769afd9b7e404e70bb8a089f355eb7cb6ffecd6f3167010f479a60f21b pushed.\n",
|
||||
"INFO:root:Finished upload of: gcr.io/zahrakubeflowcodelab/fairing-job/fairing-job:6F63F28C\n",
|
||||
"WARNING:fairing.builders.append.append:Pushed image gcr.io/zahrakubeflowcodelab/fairing-job/fairing-job:6F63F28C in 3.056034044129774s.\n"
|
||||
"Building image using Append builder...\n",
|
||||
"Creating docker context: /tmp/fairing_context_xpzlon_h\n",
|
||||
"build-train-deploy.py already exists in Fairing context, skipping...\n",
|
||||
"Loading Docker credentials for repository 'gcr.io/issue-label-bot-dev/fairing-job/fairing-job:E480ACAF'\n",
|
||||
"Invoking 'docker-credential-gcloud' to obtain Docker credentials.\n",
|
||||
"Successfully obtained Docker credentials.\n",
|
||||
"Image successfully built in 1.2515304939588532s.\n",
|
||||
"Pushing image gcr.io/issue-label-bot-dev/fairing-job/fairing-job:DA1D5CB0...\n",
|
||||
"Loading Docker credentials for repository 'gcr.io/issue-label-bot-dev/fairing-job/fairing-job:DA1D5CB0'\n",
|
||||
"Invoking 'docker-credential-gcloud' to obtain Docker credentials.\n",
|
||||
"Successfully obtained Docker credentials.\n",
|
||||
"Uploading gcr.io/issue-label-bot-dev/fairing-job/fairing-job:DA1D5CB0\n",
|
||||
"Layer sha256:9d866f8bde2a0d607a6d17edc0fbd5e00b58306efc2b0a57e0ba72f269e7c6be exists, skipping\n",
|
||||
"Layer sha256:124c757242f88002a858c23fc79f8262f9587fa30fd92507e586ad074afb42b6 exists, skipping\n",
|
||||
"Layer sha256:bbf0f5f91e8108d9b0be1ceeb749e63788ce7394a184bc8a70d24017eca7b7ba exists, skipping\n",
|
||||
"Layer sha256:e7fea64fabbc6f5961864ce5c6bcc143ab616d325b0c5a26848d8e427806104f exists, skipping\n",
|
||||
"Layer sha256:d099b15c53311dc296426716edabe61dcc19e88009c19098b17ba965357c4391 exists, skipping\n",
|
||||
"Layer sha256:079dd3e30fa3eed702bb20a2f725da9907e2732bdc4dfb2fb5084a3423c3f743 exists, skipping\n",
|
||||
"Layer sha256:80794aeb9ef80da69469ae895f20899b52d9115e4161543c83774863e97fc507 exists, skipping\n",
|
||||
"Layer sha256:eed14867f5ee443ad7efc89d0d4392683799a413244feec120f43074bc2d43ef exists, skipping\n",
|
||||
"Layer sha256:55dbf73eb7c7c005c3ccff29b62ff180e2f29245d14794dd6d5d8ad855d0ea88 exists, skipping\n",
|
||||
"Layer sha256:8ab941f264e893bf2d02a0f6d2972fa5f725995cba85b0a897cee1531525bba1 exists, skipping\n",
|
||||
"Layer sha256:2327f2e2474891211dbf7fb2d54e16e7b2889fea157b726645cc05e75ad917e8 exists, skipping\n",
|
||||
"Layer sha256:fa3f2f277e67c5cbbf1dac21dc27111a60d3cd2ef494d94aa1515d3319f2a245 exists, skipping\n",
|
||||
"Layer sha256:afde35469481d2bc446d649a7a3d099147bbf7696b66333e76a411686b617ea1 exists, skipping\n",
|
||||
"Layer sha256:d13453f7d2b8d0adfd86c3989a5b695cef5afc3efaafe559643071f258c9f06d exists, skipping\n",
|
||||
"Layer sha256:2f1ee468081da0ca09360c50281ed261d8b3fb01f664262c3f278d8619eb4e9a exists, skipping\n",
|
||||
"Layer sha256:8c58e650bb886ab24426958165c15abe1a1c10e8710f50233701fd503e23e7ac exists, skipping\n",
|
||||
"Layer sha256:507170ae8cfaca6cf2999295221d1324f1051fa15ba59e04dd7dafdc8de565bc exists, skipping\n",
|
||||
"Layer sha256:b893ca5fa31bb87be0d3fa3a403dac7ca12c955d6fd522fd35e3260dbd0e99da exists, skipping\n",
|
||||
"Layer sha256:8143617e89d7ba1957e3dc6d7093a48bd0cd4a2a709bc0c9d0ffc6dde11467e8 exists, skipping\n",
|
||||
"Layer sha256:1cf84c00b8903926c231b4b5974c0419556a4a578bf9416f585fcbf1b7aa70ab exists, skipping\n",
|
||||
"Layer sha256:4bfa6a63a3897359eff3ca3ee27c2e05ba76b790a07e6583714c1d324c2d4f21 exists, skipping\n",
|
||||
"Layer sha256:5d8a6f34a39a1e098f09b39ee4e9d4a178fef6ec71c2046fe0b040c4667c8143 exists, skipping\n",
|
||||
"Layer sha256:0ba512db704a2eb85f7f372d1c809d58589531e3bae794f0aaba86cee912f923 exists, skipping\n",
|
||||
"Layer sha256:a5ba9de0ac70b35658f5898c27b52063a597d790308fb853021e881e04a6efb7 exists, skipping\n",
|
||||
"Layer sha256:bad6918fba4b1c68f82d1a4b6063b3ce64975a73b33b38b35454b1d484a6b57b exists, skipping\n",
|
||||
"Layer sha256:0fd02182c40eb28e13c4d7efd5dd4c81d985d9b07c9c809cc26e7bdb2dced07e exists, skipping\n",
|
||||
"Layer sha256:541a15d3a9d79f7d3e5e0f552f396406b3e3093247f71e0ae71dd8b7242ec428 exists, skipping\n",
|
||||
"Layer sha256:ecc17173ccb5b7692a6d31b0077b8e4f543fb45f8c2b5c252dcad9ad0c9be0f7 exists, skipping\n",
|
||||
"Layer sha256:07e06c833ecb3b115e378d7f2ba5817ba77cfd02f5794a9817ede0622fbbf8a5 exists, skipping\n",
|
||||
"Layer sha256:9ee379bde91a3cecfb08d4189af0a2bcecc2da1c5102e49443088ccd7bd9abfa exists, skipping\n",
|
||||
"Layer sha256:90a7e2cb4d7460e55f83c6e47f9f8d089895ee6e1cc51ae5c23eab3bdcb70363 exists, skipping\n",
|
||||
"Layer sha256:398d32b153e84fe343f0c5b07d65e89b05551aae6cb8b3a03bb2b662976eb3b8 exists, skipping\n",
|
||||
"Layer sha256:3885f9a80c70bf1aa3d3b925004fcca76334d45aa96d5e95412b40cae1dbdbba pushed.\n",
|
||||
"Layer sha256:e94d45d512ce4033820c7df7dae67aa2d300528fed0ea5a53d6dcd099b2e4ca1 pushed.\n",
|
||||
"Finished upload of: gcr.io/issue-label-bot-dev/fairing-job/fairing-job:DA1D5CB0\n",
|
||||
"Pushed image gcr.io/issue-label-bot-dev/fairing-job/fairing-job:DA1D5CB0 in 3.6773080190178007s.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"preprocessor = ConvertNotebookPreprocessorWithFire(\"HousingServe\")\n",
|
||||
"\n",
|
||||
"if not preprocessor.input_files:\n",
|
||||
" preprocessor.input_files = set()\n",
|
||||
"input_files=[\"xgboost_util.py\", \"mockup-model.dat\"]\n",
|
||||
"preprocessor.input_files = set([os.path.normpath(f) for f in input_files])\n",
|
||||
"preprocessor.preprocess()\n",
|
||||
"builder = append.append.AppendBuilder(registry=DOCKER_REGISTRY,\n",
|
||||
" base_image=base_image, preprocessor=preprocessor)\n",
|
||||
" base_image=cluster_builder.image_tag, preprocessor=preprocessor)\n",
|
||||
"builder.build()\n"
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.20/kfp.tar
|
|||
def notebook_setup():
|
||||
# Install the SDK
|
||||
|
||||
subprocess.check_call(["pip3", "install", "-r", "requirements.txt"])
|
||||
subprocess.check_call(["pip3", "install", KFP_PACKAGE, "--upgrade"])
|
||||
|
||||
logging.basicConfig(format='%(message)s')
|
||||
|
|
|
|||
Loading…
Reference in New Issue