mirror of https://github.com/kubeflow/examples.git
102 lines
3.5 KiB
Plaintext
102 lines
3.5 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import kfp\n",
|
|
"from kfp import dsl\n",
|
|
"\n",
|
|
"def SendMsg():\n",
|
|
" vop = dsl.VolumeOp(name=\"pvc\",\n",
|
|
" resource_name=\"pvc\", size='1Gi', \n",
|
|
" modes=dsl.VOLUME_MODE_RWO)\n",
|
|
"\n",
|
|
" return dsl.ContainerOp(\n",
|
|
" name = 'load-data', \n",
|
|
" image = 'hubdocker76/openvaccine:v10', \n",
|
|
" command = ['python3', 'load.py'],\n",
|
|
"\n",
|
|
" pvolumes={\n",
|
|
" '/data': vop.volume\n",
|
|
" }\n",
|
|
" )\n",
|
|
"\n",
|
|
"def GetMsg(comp1):\n",
|
|
" return dsl.ContainerOp(\n",
|
|
" name = 'preprocess',\n",
|
|
" image = 'hubdocker76/preprocess-data:v10',\n",
|
|
" pvolumes={\n",
|
|
" '/data': comp1.pvolumes['/data']\n",
|
|
" },\n",
|
|
" command = ['python3', 'preprocess.py']\n",
|
|
" )\n",
|
|
"\n",
|
|
"def Train(comp2, trial, epoch, batchsize, embeddim, hiddendim, dropout, spdropout, trainsequencelength):\n",
|
|
" return dsl.ContainerOp(\n",
|
|
" name = 'train',\n",
|
|
" image = 'hubdocker76/model-training:v21',\n",
|
|
" command = ['python3', 'model.py'],\n",
|
|
" arguments=[\n",
|
|
" '--LR', trial,\n",
|
|
" '--EPOCHS', epoch,\n",
|
|
" '--BATCH_SIZE', batchsize,\n",
|
|
" '--EMBED_DIM', embeddim,\n",
|
|
" '--HIDDEN_DIM', hiddendim,\n",
|
|
" '--DROPOUT', dropout,\n",
|
|
" '--SP_DROPOUT', spdropout,\n",
|
|
" '--TRAIN_SEQUENCE_LENGTH', trainsequencelength\n",
|
|
" ],\n",
|
|
" pvolumes={\n",
|
|
" '/data': comp2.pvolumes['/data']\n",
|
|
" }\n",
|
|
" )\n",
|
|
"\n",
|
|
"def Eval(comp1, trial, epoch, batchsize, embeddim, hiddendim, dropout, spdropout, trainsequencelength):\n",
|
|
" return dsl.ContainerOp(\n",
|
|
" name = 'Evaluate',\n",
|
|
" image = 'hubdocker76/eval:v4',\n",
|
|
" arguments=[\n",
|
|
" '--LR', trial,\n",
|
|
" '--EPOCHS', epoch,\n",
|
|
" '--BATCH_SIZE', batchsize,\n",
|
|
" '--EMBED_DIM', embeddim,\n",
|
|
" '--HIDDEN_DIM', hiddendim,\n",
|
|
" '--DROPOUT', dropout,\n",
|
|
" '--SP_DROPOUT', spdropout,\n",
|
|
" '--TRAIN_SEQUENCE_LENGTH', trainsequencelength\n",
|
|
" ],\n",
|
|
" pvolumes={\n",
|
|
" '/data': comp1.pvolumes['/data']\n",
|
|
" },\n",
|
|
" command = ['python3', 'eval.py']\n",
|
|
" )\n",
|
|
"\n",
|
|
"@dsl.pipeline(\n",
|
|
" name = 'openvaccine',\n",
|
|
" description = 'pipeline to run openvaccine')\n",
|
|
"\n",
|
|
"def passing_parameter(trial, epoch, batchsize, embeddim, hiddendim, dropout, spdropout, trainsequencelength):\n",
|
|
" comp1 = SendMsg().add_pod_label(\"kaggle-secret\", \"true\")\n",
|
|
" comp2 = GetMsg(comp1)\n",
|
|
" comp3 = Train(comp2, trial, epoch, batchsize, embeddim, hiddendim, dropout, spdropout, trainsequencelength)\n",
|
|
" comp4 = Eval(comp3, trial, epoch, batchsize, embeddim, hiddendim, dropout, spdropout, trainsequencelength)\n",
|
|
"\n",
|
|
"if __name__ == '__main__':\n",
|
|
" import kfp.compiler as compiler\n",
|
|
" compiler.Compiler().compile(passing_parameter, __file__[:-3]+ '.yaml')\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"language_info": {
|
|
"name": "python"
|
|
},
|
|
"orig_nbformat": 4
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|