parameterize search index job name (#358)

* parameterize search index job name

* change namespace

* Update search-index-creator.jsonnet

code_search/kubeflow/app.yaml

@@ -6,6 +6,12 @@ environments:
       server: https://35.185.115.154
     k8sVersion: v1.10.9
     path: cs_demo
+  pipeline:
+    destination:
+      namespace: kubeflow
+      server: https://kubernetes.default
+    k8sVersion: v1.9.7
+    path: pipeline
 kind: ksonnet.io/app
 libraries:
   examples:

code_search/kubeflow/components/experiments.libsonnet

@@ -21,4 +21,12 @@
     lookupFile: "gs://code-search-demo/20181104/code-embeddings-index/embedding-to-info.csv",
     indexFile: "gs://code-search-demo/20181104/code-embeddings-index/embeddings.index",
   },
+  "pipeline": {
+    name: "pipeline",
+    project: "code-search-demo",
+
+    problem: "kf_github_function_docstring",
+    // modelBasePath shouldn't have integer in it.
+    modelBasePath: "gs://code-search-demo/models/20181107-dist-sync-gpu/export/",
+  },
 }

code_search/kubeflow/components/params.libsonnet

@@ -92,6 +92,8 @@
     },
     "search-index-creator": {
       name: "search-index-creator",
+      jobNameSuffix: "",
+      image: $.components["t2t-job"].dataflowImage,
       dataDir: $.components["t2t-code-search"].workingDir + "/data",
       lookupFile: $.components["t2t-code-search"].workingDir + "/code_search_index.csv",
       indexFile: $.components["t2t-code-search"].workingDir + "/code_search_index.nmslib",

code_search/kubeflow/components/search-index-creator.jsonnet

@@ -1,13 +1,13 @@
 local k = import "k.libsonnet";
 
 local env = std.extVar("__ksonnet/environments");
-local params = std.extVar("__ksonnet/params").components["search-index-creator"];
+local baseParams = std.extVar("__ksonnet/params").components["search-index-creator"];
 local experiments = import "experiments.libsonnet";
 
-local baseParams = std.extVar("__ksonnet/params").components["submit-code-embeddings-job"];
 local experimentName = baseParams.experiment;
+local jobNameSuffix = baseParams.jobNameSuffix;
 local params = baseParams + experiments[experimentName] + {
-  name: experimentName + "-create-search-index",
+  name: experimentName + "-create-search-index-" + jobNameSuffix,
 };
 
 local jobSpec = {

code_search/kubeflow/environments/pipeline/globals.libsonnet

@@ -0,0 +1,9 @@
+{
+  // Warning: Do not define a global "image" as that will end up overriding
+  // the image parameter for all components. Define more specific names
+  // e.g. "dataflowImage", "trainerCpuImage", "trainerGpuImage",
+  workingDir: "gs://code-search-demo/20181104",
+  dataDir: "gs://code-search-demo/20181104/data",
+  project: "code-search-demo",
+  experiment: "pipeline",
+}

code_search/kubeflow/environments/pipeline/main.jsonnet

@@ -0,0 +1,8 @@
+local base = import "base.libsonnet";
+// uncomment if you reference ksonnet-lib
+// local k = import "k.libsonnet";
+
+base + {
+  // Insert user-specified overrides here. For example if a component is named \"nginx-deployment\", you might have something like:\n")
+  // "nginx-deployment"+: k.deployment.mixin.metadata.labels({foo: "bar"})
+}

code_search/kubeflow/environments/pipeline/params.libsonnet

@@ -0,0 +1,12 @@
+local params = std.extVar("__ksonnet/params");
+local globals = import "globals.libsonnet";
+local envParams = params + {
+  components +: {
+  },
+};
+
+{
+  components: {
+    [x]: envParams.components[x] + globals, for x in std.objectFields(envParams.components)
+  },
+}