mirror of https://github.com/kubeflow/examples.git
Allow ks param set for Code Search Ksonnet Application (#224)
* Allow ks param set for t2t-code-search * Update notebook with working directory param set * Abstract out common variables for easy ks param set
This commit is contained in:
Sanyam Kapoor
k8s-ci-robot
parent
f3806d0bac
commit
8fce4a7799
|
|
@ -79,7 +79,7 @@
|
|||
"CLUSTER_NAMESPACE = 'kubeflow-latest'\n",
|
||||
"\n",
|
||||
"TARGET_DATASET = 'code_search'\n",
|
||||
"WORKING_DIR = 'gs://kubeflow-examples/t2t-code-search/20180813'\n",
|
||||
"WORKING_DIR = 'gs://kubeflow-examples/t2t-code-search/notebook-demo'\n",
|
||||
"WORKER_MACHINE_TYPE = 'n1-highcpu-32'\n",
|
||||
"NUM_WORKERS = 16\n",
|
||||
"\n",
|
||||
|
|
@ -141,7 +141,11 @@
|
|||
"\n",
|
||||
"cd kubeflow\n",
|
||||
"\n",
|
||||
"ks env add code-search --context=$(kubectl config current-context)"
|
||||
"# Update Ksonnet application to the context set earlier\n",
|
||||
"ks env add code-search --context=$(kubectl config current-context)\n",
|
||||
"\n",
|
||||
"# Update the Working Directory of the application\n",
|
||||
"ks param set t2t-code-search workingDir ${WORKING_DIR}"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -276,8 +280,8 @@
|
|||
" --target_dataset \"${TARGET_DATASET}\" \\\n",
|
||||
" --data_dir \"${WORKING_DIR}/data\" \\\n",
|
||||
" --job_name \"${JOB_NAME}\" \\\n",
|
||||
" --temp_location \"${WORKING_DIR}/data/dataflow/temp\" \\\n",
|
||||
" --staging_location \"${WORKING_DIR}/data/dataflow/staging\" \\\n",
|
||||
" --temp_location \"${WORKING_DIR}/dataflow/temp\" \\\n",
|
||||
" --staging_location \"${WORKING_DIR}/dataflow/staging\" \\\n",
|
||||
" --worker_machine_type \"${WORKER_MACHINE_TYPE}\" \\\n",
|
||||
" --num_workers \"${NUM_WORKERS}\""
|
||||
]
|
||||
|
|
@ -416,6 +420,7 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# WARNING: This routine will fail if no export has been completed successfully.\n",
|
||||
"MODEL_VERSION = max([int(ts[:-1]) for ts in EXPORT_DIR_LS.split('\\n') if ts])\n",
|
||||
"\n",
|
||||
"# DO NOT MODIFY. These are environment variables to be used in a bash shell.\n",
|
||||
|
|
@ -447,8 +452,8 @@
|
|||
" --data_dir \"${WORKING_DIR}/data\" \\\n",
|
||||
" --saved_model_dir \"${WORKING_DIR}/output/export/Servo/${MODEL_VERSION}\" \\\n",
|
||||
" --job_name compute-function-embeddings\n",
|
||||
" --temp_location \"${WORKING_DIR}/data/dataflow/temp\" \\\n",
|
||||
" --staging_location \"${WORKING_DIR}/data/dataflow/staging\" \\\n",
|
||||
" --temp_location \"${WORKING_DIR}/dataflow/temp\" \\\n",
|
||||
" --staging_location \"${WORKING_DIR}/dataflow/staging\" \\\n",
|
||||
" --worker_machine_type \"${WORKER_MACHINE_TYPE}\" \\\n",
|
||||
" --num_workers \"${NUM_WORKERS}\""
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,103 +1,82 @@
|
|||
{
|
||||
global: {
|
||||
// User-defined global parameters; accessible to all component and environments, Ex:
|
||||
// replicas: 4,
|
||||
t2tWorkingDir: "gs://kubeflow-examples/t2t-code-search/20180802",
|
||||
},
|
||||
global: {},
|
||||
components: {
|
||||
// Component-level parameters, defined initially from 'ks prototype use ...'
|
||||
// Each object below should correspond to a component in the components/ directory
|
||||
"t2t-job": {
|
||||
jobType: "trainer",
|
||||
|
||||
jobType: 'trainer',
|
||||
numWorker: 1,
|
||||
numPs: 0,
|
||||
numWorkerGpu: 0,
|
||||
numPsGpu: 0,
|
||||
|
||||
train_steps: 100,
|
||||
eval_steps: 10,
|
||||
|
||||
image: "gcr.io/kubeflow-dev/code-search:v20180802-c622aac",
|
||||
imageGpu: "gcr.io/kubeflow-dev/code-search:v20180802-c622aac-gpu",
|
||||
image: 'gcr.io/kubeflow-dev/code-search:v20180802-c622aac',
|
||||
imageGpu: 'gcr.io/kubeflow-dev/code-search:v20180802-c622aac-gpu',
|
||||
imagePullSecrets: [],
|
||||
|
||||
dataDir: "null",
|
||||
outputDir: "null",
|
||||
model: "null",
|
||||
hparams_set: "null",
|
||||
dataDir: 'null',
|
||||
outputDir: 'null',
|
||||
model: 'null',
|
||||
hparams_set: 'null',
|
||||
},
|
||||
"t2t-code-search": {
|
||||
workingDir: 'gs://example/prefix',
|
||||
problem: 'github_function_docstring',
|
||||
model: 'similarity_transformer',
|
||||
hparams_set: 'transformer_tiny',
|
||||
},
|
||||
|
||||
"t2t-code-search-datagen": {
|
||||
jobType: "datagen",
|
||||
|
||||
name: "t2t-code-search-datagen",
|
||||
|
||||
problem: "github_function_docstring",
|
||||
dataDir: $.global.t2tWorkingDir + "/data",
|
||||
jobType: 'datagen',
|
||||
name: 't2t-code-search-datagen',
|
||||
problem: $.components["t2t-code-search"].problem,
|
||||
dataDir: $.components['t2t-code-search'].workingDir + '/data',
|
||||
},
|
||||
|
||||
"t2t-code-search-trainer": {
|
||||
jobType: "trainer",
|
||||
|
||||
name: "t2t-code-search-trainer",
|
||||
|
||||
problem: "github_function_docstring",
|
||||
dataDir: $.global.t2tWorkingDir + "/data",
|
||||
outputDir: $.global.t2tWorkingDir + "/output",
|
||||
model: "similarity_transformer",
|
||||
hparams_set: "transformer_tiny",
|
||||
jobType: 'trainer',
|
||||
name: 't2t-code-search-trainer',
|
||||
problem: $.components["t2t-code-search"].problem,
|
||||
dataDir: $.components['t2t-code-search'].workingDir + '/data',
|
||||
outputDir: $.components['t2t-code-search'].workingDir + '/output',
|
||||
model: $.components["t2t-code-search"].model,
|
||||
hparams_set: $.components["t2t-code-search"].hparams_set,
|
||||
},
|
||||
|
||||
"t2t-code-search-exporter": {
|
||||
jobType: "exporter",
|
||||
|
||||
name: "t2t-code-search-exporter",
|
||||
|
||||
problem: "github_function_docstring",
|
||||
dataDir: $.global.t2tWorkingDir + "/data",
|
||||
outputDir: $.global.t2tWorkingDir + "/output",
|
||||
model: "similarity_transformer",
|
||||
hparams_set: "transformer_tiny",
|
||||
jobType: 'exporter',
|
||||
name: 't2t-code-search-exporter',
|
||||
problem: $.components["t2t-code-search"].problem,
|
||||
dataDir: $.components['t2t-code-search'].workingDir + '/data',
|
||||
outputDir: $.components['t2t-code-search'].workingDir + '/output',
|
||||
model: $.components["t2t-code-search"].model,
|
||||
hparams_set: $.components["t2t-code-search"].hparams_set,
|
||||
},
|
||||
|
||||
"t2t-code-search-serving": {
|
||||
name: "t2t-code-search",
|
||||
|
||||
modelName: "t2t-code-search",
|
||||
modelPath: $.global.t2tWorkingDir + "/output/export/Servo",
|
||||
modelServerImage: "gcr.io/kubeflow-images-public/tensorflow-serving-1.8:latest",
|
||||
cloud: "gcp",
|
||||
gcpCredentialSecretName: "user-gcp-sa",
|
||||
name: 't2t-code-search',
|
||||
modelName: 't2t-code-search',
|
||||
modelPath: $.components['t2t-code-search'].workingDir + '/output/export/Servo',
|
||||
modelServerImage: 'gcr.io/kubeflow-images-public/tensorflow-serving-1.8:latest',
|
||||
cloud: 'gcp',
|
||||
gcpCredentialSecretName: 'user-gcp-sa',
|
||||
},
|
||||
|
||||
"nmslib": {
|
||||
nmslib: {
|
||||
replicas: 1,
|
||||
image: "gcr.io/kubeflow-dev/code-search-ui:v20180806-7b0fcaa",
|
||||
|
||||
problem: "null",
|
||||
dataDir: "null",
|
||||
lookupFile: "null",
|
||||
indexFile: "null",
|
||||
servingUrl: "null",
|
||||
image: 'gcr.io/kubeflow-dev/code-search-ui:v20180806-7b0fcaa',
|
||||
problem: 'null',
|
||||
dataDir: 'null',
|
||||
lookupFile: 'null',
|
||||
indexFile: 'null',
|
||||
servingUrl: 'null',
|
||||
},
|
||||
|
||||
"search-index-creator": {
|
||||
name: "search-index-creator",
|
||||
|
||||
dataDir: $.global.t2tWorkingDir + "/data",
|
||||
lookupFile: $.global.t2tWorkingDir + "/code_search_index.csv",
|
||||
indexFile: $.global.t2tWorkingDir + "/code_search_index.nmslib",
|
||||
name: 'search-index-creator',
|
||||
dataDir: $.components['t2t-code-search'].workingDir + '/data',
|
||||
lookupFile: $.components['t2t-code-search'].workingDir + '/code_search_index.csv',
|
||||
indexFile: $.components['t2t-code-search'].workingDir + '/code_search_index.nmslib',
|
||||
},
|
||||
|
||||
"search-index-server": {
|
||||
name: "search-index-server",
|
||||
|
||||
problem: "github_function_docstring",
|
||||
dataDir: $.global.t2tWorkingDir + "/data",
|
||||
lookupFile: $.global.t2tWorkingDir + "/code_search_index.csv",
|
||||
indexFile: $.global.t2tWorkingDir + "/code_search_index.nmslib",
|
||||
servingUrl: "http://t2t-code-search.kubeflow:8000/v1/models/t2t-code-search:predict",
|
||||
name: 'search-index-server',
|
||||
problem: $.components["t2t-code-search"].problem,
|
||||
dataDir: $.components['t2t-code-search'].workingDir + '/data',
|
||||
lookupFile: $.components['t2t-code-search'].workingDir + '/code_search_index.csv',
|
||||
indexFile: $.components['t2t-code-search'].workingDir + '/code_search_index.nmslib',
|
||||
servingUrl: 'http://t2t-code-search.kubeflow:8000/v1/models/t2t-code-search:predict',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
// NOTE: This is only a dummy to allow `ks param set`. DONOT use.
|
||||
|
||||
local k = import "k.libsonnet";
|
||||
local t2tJob = import "t2t-job.libsonnet";
|
||||
|
||||
local env = std.extVar("__ksonnet/environments");
|
||||
local params = std.extVar("__ksonnet/params").components["t2t-code-search"];
|
||||
|
||||
std.prune(k.core.v1.list.new([t2tJob.parts(params, env).job]))
|
||||
Loading…
Reference in New Issue