diff --git a/code_search/demo/cs-demo-1103/ks_app/app.yaml b/code_search/demo/cs-demo-1103/ks_app/app.yaml index f3f7a0a6..c1f8aad7 100644 --- a/code_search/demo/cs-demo-1103/ks_app/app.yaml +++ b/code_search/demo/cs-demo-1103/ks_app/app.yaml @@ -1,4 +1,4 @@ -apiVersion: 0.1.0 +apiVersion: 0.3.0 environments: default: destination: @@ -8,38 +8,50 @@ environments: path: default kind: ksonnet.io/app libraries: - argo: + kfp/pipeline: + name: pipeline + registry: kfp + version: af3376fee4229507acc3a5f3af8b129c9a48a2dc + kubeflow/argo: name: argo registry: kubeflow - core: + version: "" + kubeflow/core: name: core registry: kubeflow - examples: + version: "" + kubeflow/examples: name: examples registry: kubeflow - katib: + version: "" + kubeflow/katib: name: katib registry: kubeflow - mpi-job: + version: "" + kubeflow/mpi-job: name: mpi-job registry: kubeflow - pytorch-job: + version: "" + kubeflow/pytorch-job: name: pytorch-job registry: kubeflow - seldon: + version: "" + kubeflow/seldon: name: seldon registry: kubeflow - tf-serving: + version: "" + kubeflow/tf-serving: name: tf-serving registry: kubeflow + version: "" name: ks_app registries: incubator: - gitVersion: - commitSha: 40285d8a14f1ac5787e405e1023cf0c07f6aa28c - refSpec: master protocol: github uri: github.com/ksonnet/parts/tree/master/incubator + kfp: + protocol: github + uri: https://github.com/kubeflow/kubeflow/tree/master/kubeflow kubeflow: protocol: fs uri: /home/jlewi/git_kubeflow/kubeflow diff --git a/code_search/demo/cs-demo-1103/ks_app/components/params.libsonnet b/code_search/demo/cs-demo-1103/ks_app/components/params.libsonnet index 6f725b25..7d5055ee 100644 --- a/code_search/demo/cs-demo-1103/ks_app/components/params.libsonnet +++ b/code_search/demo/cs-demo-1103/ks_app/components/params.libsonnet @@ -105,5 +105,12 @@ privateGKECluster: "false", secretName: "envoy-ingress-tls", }, + pipeline: { + apiImage: "gcr.io/ml-pipeline/api-server:0.1.2", + name: "pipeline", + persistenceAgentImage: "gcr.io/ml-pipeline/persistenceagent:0.1.2", + scheduledWorkflowImage: "gcr.io/ml-pipeline/scheduledworkflow:0.1.2", + uiImage: "gcr.io/ml-pipeline/frontend:0.1.2", + }, }, } diff --git a/code_search/demo/cs-demo-1103/ks_app/components/pipeline.jsonnet b/code_search/demo/cs-demo-1103/ks_app/components/pipeline.jsonnet new file mode 100644 index 00000000..382ca684 --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/components/pipeline.jsonnet @@ -0,0 +1,7 @@ +local env = std.extVar("__ksonnet/environments"); +local params = std.extVar("__ksonnet/params").components.pipeline; + +local k = import "k.libsonnet"; +local all = import "kfp/pipeline/all.libsonnet"; + +std.prune(k.core.v1.list.new(all.parts(env, params).all)) diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/OWNERS b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/OWNERS new file mode 100644 index 00000000..79b074bb --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/OWNERS @@ -0,0 +1,3 @@ +approvers: + - IronPan +reviewers: diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/all.libsonnet b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/all.libsonnet new file mode 100644 index 00000000..83b5112d --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/all.libsonnet @@ -0,0 +1,25 @@ +{ + parts(_env, _params):: { + local params = _env + _params, + + local minio = import "kfp/pipeline/minio.libsonnet", + local mysql = import "kfp/pipeline/mysql.libsonnet", + local pipeline_apiserver = import "kfp/pipeline/pipeline-apiserver.libsonnet", + local pipeline_scheduledworkflow = import "kfp/pipeline/pipeline-scheduledworkflow.libsonnet", + local pipeline_persistenceagent = import "kfp/pipeline/pipeline-persistenceagent.libsonnet", + local pipeline_ui = import "kfp/pipeline/pipeline-ui.libsonnet", + + local name = params.name, + local namespace = params.namespace, + local apiImage = params.apiImage, + local scheduledWorkflowImage = params.scheduledWorkflowImage, + local persistenceAgentImage = params.persistenceAgentImage, + local uiImage = params.uiImage, + all:: minio.parts(namespace).all + + mysql.parts(namespace).all + + pipeline_apiserver.all(namespace, apiImage) + + pipeline_scheduledworkflow.all(namespace, scheduledWorkflowImage) + + pipeline_persistenceagent.all(namespace, persistenceAgentImage) + + pipeline_ui.all(namespace, uiImage), + }, +} diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/minio.libsonnet b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/minio.libsonnet new file mode 100644 index 00000000..788d37f6 --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/minio.libsonnet @@ -0,0 +1,131 @@ +{ + parts(namespace):: { + all:: [ + $.parts(namespace).pvc, + $.parts(namespace).service, + $.parts(namespace).deploy, + $.parts(namespace).secret, + ], + + pvc: { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: "minio-pv-claim", + namespace: namespace, + }, + spec: { + accessModes: [ + "ReadWriteOnce", + ], + resources: { + requests: { + storage: "10Gi", + }, + }, + }, + }, //pvc + + service: { + apiVersion: "v1", + kind: "Service", + metadata: { + name: "minio-service", + namespace: namespace, + }, + spec: { + ports: [ + { + port: 9000, + targetPort: 9000, + protocol: "TCP", + }, + ], + selector: { + app: "minio", + }, + }, + status: { + loadBalancer: {}, + }, + }, //service + + deploy: { + apiVersion: "apps/v1beta1", + kind: "Deployment", + metadata: { + name: "minio", + namespace: namespace, + }, + spec: { + strategy: { + type: "Recreate", + }, + template: { + metadata: { + labels: { + app: "minio", + }, + }, + spec: { + volumes: [ + { + name: "data", + persistentVolumeClaim: { + claimName: "minio-pv-claim", + }, + }, + ], + containers: [ + { + name: "minio", + volumeMounts: [ + { + name: "data", + mountPath: "/data", + }, + ], + image: "minio/minio:RELEASE.2018-02-09T22-40-05Z", + args: [ + "server", + "/data", + ], + env: [ + { + name: "MINIO_ACCESS_KEY", + value: "minio", + }, + { + name: "MINIO_SECRET_KEY", + value: "minio123", + }, + ], + ports: [ + { + containerPort: 9000, + }, + ], + }, + ], + }, + }, + }, + }, // deploy + + // The motivation behind the minio secret creation is that argo workflows depend on this secret to + // store the artifact in minio. + secret: { + apiVersion: "v1", + kind: "Secret", + metadata: { + name: "mlpipeline-minio-artifact", + namespace: namespace, + }, + type: "Opaque", + data: { + accesskey: std.base64("minio"), + secretkey: std.base64("minio123"), + }, + }, // secret + }, // parts +} diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/mysql.libsonnet b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/mysql.libsonnet new file mode 100644 index 00000000..79a4b624 --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/mysql.libsonnet @@ -0,0 +1,110 @@ +{ + parts(namespace):: { + all:: [ + $.parts(namespace).pvc, + $.parts(namespace).service, + $.parts(namespace).deploy, + ], + + pvc: { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: "mysql-pv-claim", + namespace: namespace, + }, + spec: { + accessModes: [ + "ReadWriteOnce", + ], + resources: { + requests: { + storage: "10Gi", + }, + }, + }, + }, //pvc + + service: { + apiVersion: "v1", + kind: "Service", + metadata: { + name: "mysql", + namespace: namespace, + }, + spec: { + ports: [ + { + port: 3306, + }, + ], + selector: { + app: "mysql", + }, + }, + status: { + loadBalancer: {}, + }, + }, //service + + deploy: { + apiVersion: "apps/v1beta2", + kind: "Deployment", + metadata: { + name: "mysql", + namespace: namespace, + }, + spec: { + selector: { + matchLabels: { + app: "mysql", + }, + }, + strategy: { + type: "Recreate", + }, + template: { + metadata: { + labels: { + app: "mysql", + }, + }, + spec: { + containers: [ + { + image: "mysql:5.6", + name: "mysql", + env: [ + { + name: "MYSQL_ALLOW_EMPTY_PASSWORD", + value: "true", + }, + ], + ports: [ + { + containerPort: 3306, + name: "mysql", + }, + ], + volumeMounts: [ + { + name: "mysql-persistent-storage", + mountPath: "/var/lib/mysql", + }, + ], + }, + ], + volumes: [ + { + name: "mysql-persistent-storage", + persistentVolumeClaim: { + claimName: "mysql-pv-claim", + }, + }, + ], + }, + }, + }, + }, //deploy + }, //parts +} diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/parts.yaml b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/parts.yaml new file mode 100644 index 00000000..7a91ee5f --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/parts.yaml @@ -0,0 +1,47 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "name": "pipeline", + "apiVersion": "0.0.1", + "kind": "ksonnet.io/parts", + "description": "Prototypes for running Kubeflow pipeline.\n", + "author": "kubeflow-team ", + "contributors": [ + { + "name": "Yang Pan", + "email": "yangpa@google.com" + } + ], + "repository": { + "type": "git", + "url": "https://github.com/kubeflow/pipelines" + }, + "bugs": { + "url": "https://github.com/kubeflow/pipelines/issues" + }, + "keywords": [ + "pipeline" + ], + "quickStart": { + "prototype": "io.ksonnet.pkg.pipeline", + "componentName": "pipeline", + "flags": { + "name": "pipeline", + "namespace": "default" + }, + "comment": "Deploy Kubeflow pipeline" + }, + "license": "Apache 2.0" +} diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-apiserver.libsonnet b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-apiserver.libsonnet new file mode 100644 index 00000000..d562b990 --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-apiserver.libsonnet @@ -0,0 +1,327 @@ +{ + all(namespace, apiImage):: [ + $.parts(namespace).serviceAccount, + $.parts(namespace).roleBinding, + $.parts(namespace).role, + $.parts(namespace).service, + $.parts(namespace).deploy(apiImage), + $.parts(namespace).pipelineRunnerServiceAccount, + $.parts(namespace).pipelineRunnerRole, + $.parts(namespace).pipelineRunnerRoleBinding, + ], + + parts(namespace):: { + serviceAccount: { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: "ml-pipeline", + namespace: namespace, + }, + }, // service account + + roleBinding:: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "RoleBinding", + metadata: { + labels: { + app: "ml-pipeline", + }, + name: "ml-pipeline", + namespace: namespace, + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "Role", + name: "ml-pipeline", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "ml-pipeline", + namespace: namespace, + }, + ], + }, // role binding + + role: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "Role", + metadata: { + labels: { + app: "ml-pipeline", + }, + name: "ml-pipeline", + namespace: namespace, + }, + rules: [ + { + apiGroups: [ + "argoproj.io", + ], + resources: [ + "workflows", + ], + verbs: [ + "create", + "get", + "list", + "watch", + "update", + "patch", + "delete", + ], + }, + { + apiGroups: [ + "kubeflow.org", + ], + resources: [ + "scheduledworkflows", + ], + verbs: [ + "create", + "get", + "list", + "update", + "patch", + "delete", + ], + }, + ], + }, // role + + service: { + apiVersion: "v1", + kind: "Service", + metadata: { + labels: { + app: "ml-pipeline", + }, + name: "ml-pipeline", + namespace: namespace, + }, + spec: { + ports: [ + { + port: 8888, + targetPort: 8888, + protocol: "TCP", + name: "http", + }, + { + port: 8887, + targetPort: 8887, + protocol: "TCP", + name: "grpc", + }, + ], + selector: { + app: "ml-pipeline", + }, + }, + status: { + loadBalancer: {}, + }, + }, //service + + deploy(image): { + apiVersion: "apps/v1beta2", + kind: "Deployment", + metadata: { + labels: { + app: "ml-pipeline", + }, + name: "ml-pipeline", + namespace: namespace, + }, + spec: { + selector: { + matchLabels: { + app: "ml-pipeline", + }, + }, + template: { + metadata: { + labels: { + app: "ml-pipeline", + }, + }, + spec: { + containers: [ + { + name: "ml-pipeline-api-server", + image: image, + imagePullPolicy: "Always", + ports: [ + { + containerPort: 8888, + }, + { + containerPort: 8887, + }, + ], + env: [ + { + name: "POD_NAMESPACE", + valueFrom: { + fieldRef: { + fieldPath: "metadata.namespace", + }, + }, + }, + ], + }, + ], + serviceAccountName: "ml-pipeline", + }, + }, + }, + }, // deploy + + pipelineRunnerServiceAccount: { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: "pipeline-runner", + namespace: namespace, + }, + }, // service account + + + // Grant admin permission so the pipeline can launch any resource in the cluster. + pipelineRunnerRole: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "ClusterRole", + metadata: { + labels: { + app: "pipeline-runner", + }, + name: "pipeline-runner", + namespace: namespace, + }, + rules: [ + { + apiGroups: [""], + resources: [ + "secrets", + ], + verbs: [ + "get", + ], + }, + { + apiGroups: [""], + resources: [ + "configmaps", + ], + verbs: [ + "get", + "watch", + "list", + ], + }, + { + apiGroups: [ + "", + ], + resources: [ + "persistentvolumeclaims", + ], + verbs: [ + "create", + "delete", + ], + }, + { + apiGroups: [ + "argoproj.io", + ], + resources: [ + "workflows", + ], + verbs: [ + "get", + "list", + "watch", + "update", + "patch", + ], + }, + { + apiGroups: [ + "", + ], + resources: [ + "pods", + "pods/exec", + "services", + ], + verbs: [ + "*", + ], + }, + { + apiGroups: [ + "", + "apps", + "extensions", + ], + resources: [ + "deployments", + "replicasets", + ], + verbs: [ + "*", + ], + }, + { + apiGroups: [ + "kubeflow.org", + ], + resources: [ + "*", + ], + verbs: [ + "*", + ], + }, + { + apiGroups: [ + "batch", + ], + resources: [ + "jobs", + ], + verbs: [ + "*", + ], + }, + ], + }, // operator-role + + pipelineRunnerRoleBinding:: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "ClusterRoleBinding", + metadata: { + labels: { + app: "pipeline-runner", + }, + name: "pipeline-runner", + namespace: namespace, + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "ClusterRole", + name: "pipeline-runner", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "pipeline-runner", + namespace: namespace, + }, + ], + }, // role binding + }, // parts +} diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-persistenceagent.libsonnet b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-persistenceagent.libsonnet new file mode 100644 index 00000000..8fb2f6d3 --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-persistenceagent.libsonnet @@ -0,0 +1,129 @@ +{ + all(namespace, persistenceAgentImage):: [ + $.parts(namespace).serviceAccount, + $.parts(namespace).roleBinding, + $.parts(namespace).role, + $.parts(namespace).deploy(persistenceAgentImage), + ], + + parts(namespace):: { + serviceAccount: { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: "ml-pipeline-persistenceagent", + namespace: namespace, + }, + }, // service account + + roleBinding:: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "ClusterRoleBinding", + metadata: { + labels: { + app: "ml-pipeline-persistenceagent", + }, + name: "ml-pipeline-persistenceagent", + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "ClusterRole", + // TODO: These permissions are too broad. This must be fixed. + name: "cluster-admin", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "ml-pipeline-persistenceagent", + namespace: namespace, + }, + ], + }, // role binding + + role: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "ClusterRole", + metadata: { + labels: { + app: "ml-pipeline-persistenceagent", + }, + name: "ml-pipeline-persistenceagent", + namespace: namespace, + }, + rules: [ + { + apiGroups: [ + "argoproj.io", + ], + resources: [ + "workflows", + ], + verbs: [ + "get", + "list", + "watch", + ], + }, + { + apiGroups: [ + "kubeflow.org", + ], + resources: [ + "scheduledworkflows", + ], + verbs: [ + "get", + "list", + "watch", + ], + }, + ], + }, // role + + deploy(image): { + apiVersion: "apps/v1beta2", + kind: "Deployment", + metadata: { + labels: { + app: "ml-pipeline-persistenceagent", + }, + name: "ml-pipeline-persistenceagent", + namespace: namespace, + }, + spec: { + selector: { + matchLabels: { + app: "ml-pipeline-persistenceagent", + }, + }, + template: { + metadata: { + labels: { + app: "ml-pipeline-persistenceagent", + }, + }, + spec: { + containers: [ + { + name: "ml-pipeline-persistenceagent", + image: image, + imagePullPolicy: "Always", + env: [ + { + name: "POD_NAMESPACE", + valueFrom: { + fieldRef: { + fieldPath: "metadata.namespace", + }, + }, + }, + ], + }, + ], + serviceAccountName: "ml-pipeline-persistenceagent", + }, + }, + }, + }, // deploy + }, // parts +} diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-scheduledworkflow.libsonnet b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-scheduledworkflow.libsonnet new file mode 100644 index 00000000..18f1e4d9 --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-scheduledworkflow.libsonnet @@ -0,0 +1,159 @@ +{ + all(namespace, scheduledWorkflowImage):: [ + $.parts(namespace).serviceAccount, + $.parts(namespace).roleBinding, + $.parts(namespace).role, + $.parts(namespace).deploy(scheduledWorkflowImage), + $.parts(namespace).crd, + ], + + parts(namespace):: { + serviceAccount: { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: "ml-pipeline-scheduledworkflow", + namespace: namespace, + }, + }, // service account + + roleBinding:: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "ClusterRoleBinding", + metadata: { + labels: { + app: "ml-pipeline-scheduledworkflow", + }, + name: "ml-pipeline-scheduledworkflow", + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "ClusterRole", + // TODO: These permissions are too broad. This must be fixed. + name: "cluster-admin", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "ml-pipeline-scheduledworkflow", + namespace: namespace, + }, + ], + }, // role binding + + role: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "Role", + metadata: { + labels: { + app: "ml-pipeline-scheduledworkflow", + }, + name: "ml-pipeline-scheduledworkflow", + namespace: namespace, + }, + rules: [ + { + apiGroups: [ + "argoproj.io", + ], + resources: [ + "workflows", + ], + verbs: [ + "create", + "get", + "list", + "watch", + "update", + "patch", + "delete", + ], + }, + { + apiGroups: [ + "kubeflow.org", + ], + resources: [ + "scheduledworkflows", + ], + verbs: [ + "create", + "get", + "list", + "watch", + "update", + "patch", + "delete", + ], + }, + ], + }, // role + + deploy(image): { + apiVersion: "apps/v1beta2", + kind: "Deployment", + metadata: { + labels: { + app: "ml-pipeline-scheduledworkflow", + }, + name: "ml-pipeline-scheduledworkflow", + namespace: namespace, + }, + spec: { + selector: { + matchLabels: { + app: "ml-pipeline-scheduledworkflow", + }, + }, + template: { + metadata: { + labels: { + app: "ml-pipeline-scheduledworkflow", + }, + }, + spec: { + containers: [ + { + name: "ml-pipeline-scheduledworkflow", + image: image, + imagePullPolicy: "Always", + env: [ + { + name: "POD_NAMESPACE", + valueFrom: { + fieldRef: { + fieldPath: "metadata.namespace", + }, + }, + }, + ], + }, + ], + serviceAccountName: "ml-pipeline-scheduledworkflow", + }, + }, + }, + }, // deploy + crd: { + apiVersion: "apiextensions.k8s.io/v1beta1", + kind: "CustomResourceDefinition", + metadata: { + name: "scheduledworkflows.kubeflow.org", + }, + spec: { + group: "kubeflow.org", + names: { + kind: "ScheduledWorkflow", + listKind: "ScheduledWorkflowList", + plural: "scheduledworkflows", + shortNames: [ + "swf", + ], + singular: "scheduledworkflow", + }, + scope: "Namespaced", + version: "v1alpha1", + }, + }, // crd + }, // parts +} diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-ui.libsonnet b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-ui.libsonnet new file mode 100644 index 00000000..61bd54ee --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/pipeline-ui.libsonnet @@ -0,0 +1,189 @@ +{ + all(namespace, uiImage):: [ + $.parts(namespace).serviceAccount, + $.parts(namespace).serviceUi, + $.parts(namespace).tensorboardData, + $.parts(namespace).roleBinding, + $.parts(namespace).role, + $.parts(namespace).deployUi(uiImage), + ], + parts(namespace):: { + serviceAccount: { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: "ml-pipeline-ui", + namespace: namespace, + }, + }, // service account + + serviceUi: { + apiVersion: "v1", + kind: "Service", + metadata: { + labels: { + app: "ml-pipeline-ui", + }, + name: "ml-pipeline-ui", + namespace: namespace, + annotations: { + "getambassador.io/config": + std.join("\n", [ + "---", + "apiVersion: ambassador/v0", + "kind: Mapping", + "name: pipelineui-mapping", + "prefix: /pipeline", + "rewrite: /pipeline", + "timeout_ms: 300000", + "service: ml-pipeline-ui." + namespace, + "use_websocket: true", + ]), + }, //annotations + }, + spec: { + ports: [ + { + port: 80, + targetPort: 3000, + }, + ], + selector: { + app: "ml-pipeline-ui", + }, + }, + status: { + loadBalancer: {}, + }, + }, //serviceUi + + tensorboardData: { + apiVersion: "v1", + kind: "Service", + metadata: { + labels: { + app: "ml-pipeline-tensorboard-ui", + }, + name: "ml-pipeline-tensorboard-ui", + namespace: namespace, + annotations: { + "getambassador.io/config": + std.join("\n", [ + "---", + "apiVersion: ambassador/v0", + "kind: Mapping", + "name: pipeline-tensorboard-ui-mapping", + "prefix: /data", + "rewrite: /data", + "timeout_ms: 300000", + "service: ml-pipeline-ui." + namespace, + "use_websocket: true", + ]), + }, //annotations + }, + spec: { + ports: [ + { + port: 80, + targetPort: 3000, + }, + ], + selector: { + app: "ml-pipeline-tensorboard-ui", + }, + }, + status: { + loadBalancer: {}, + }, + }, //tensorboardData + + roleBinding:: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "RoleBinding", + metadata: { + labels: { + app: "ml-pipeline-ui", + }, + name: "ml-pipeline-ui", + namespace: namespace, + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "Role", + name: "ml-pipeline-ui", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "ml-pipeline-ui", + namespace: namespace, + }, + ], + }, // role binding + + role: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "Role", + metadata: { + labels: { + app: "ml-pipeline-ui", + }, + name: "ml-pipeline-ui", + namespace: namespace, + }, + rules: [ + { + apiGroups: [""], + resources: [ + "pods", + "pods/log", + ], + verbs: [ + "create", + "get", + "list", + ], + }, + ], + }, // role + + deployUi(image): { + apiVersion: "apps/v1beta2", + kind: "Deployment", + metadata: { + labels: { + app: "ml-pipeline-ui", + }, + name: "ml-pipeline-ui", + namespace: namespace, + }, + spec: { + selector: { + matchLabels: { + app: "ml-pipeline-ui", + }, + }, + template: { + metadata: { + labels: { + app: "ml-pipeline-ui", + }, + }, + spec: { + containers: [ + { + name: "ml-pipeline-ui", + image: image, + imagePullPolicy: "Always", + ports: [{ + containerPort: 3000, + }], + }, + ], + serviceAccountName: "ml-pipeline-ui", + }, + }, + }, + }, // deployUi + }, // parts +} diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/prototypes/pipeline.jsonnet b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/prototypes/pipeline.jsonnet new file mode 100644 index 00000000..29d3b932 --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/prototypes/pipeline.jsonnet @@ -0,0 +1,14 @@ +// @apiVersion 0.1 +// @name io.ksonnet.pkg.pipeline +// @description a Kubeflow pipeline deployment. +// @shortDescription Kubeflow pipeline +// @param name string Name to give to each of the components +// @optionalParam apiImage string gcr.io/ml-pipeline/api-server:0.1.2 API docker image +// @optionalParam scheduledWorkflowImage string gcr.io/ml-pipeline/scheduledworkflow:0.1.2 schedule workflow docker image +// @optionalParam persistenceAgentImage string gcr.io/ml-pipeline/persistenceagent:0.1.2 persistence agent docker image +// @optionalParam uiImage string gcr.io/ml-pipeline/frontend:0.1.2 UI docker image + +local k = import "k.libsonnet"; +local all = import "kfp/pipeline/all.libsonnet"; + +std.prune(k.core.v1.list.new(all.parts(env, params).all)) diff --git a/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/spartakus.libsonnet b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/spartakus.libsonnet new file mode 100644 index 00000000..b7b25412 --- /dev/null +++ b/code_search/demo/cs-demo-1103/ks_app/vendor/kfp/pipeline@af3376fee4229507acc3a5f3af8b129c9a48a2dc/spartakus.libsonnet @@ -0,0 +1,99 @@ +{ + all(namespace, usageId):: [ + $.parts(namespace).serviceAccount, + $.parts(namespace).clusterRole, + $.parts(namespace).clusterRoleBinding, + $.parts(namespace).deployVolunteer(usageId), + ], + + parts(namespace):: { + serviceAccount: { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: "spartakus", + namespace: namespace, + }, + }, // service account + clusterRole: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "ClusterRole", + metadata: { + name: "spartakus", + }, + rules: [ + { + apiGroups: [ + "", + ], + resources: [ + "nodes", + ], + verbs: [ + "get", + "list", + ], + }, + ], + }, // clusterRole + clusterRoleBinding: { + apiVersion: "rbac.authorization.k8s.io/v1beta1", + kind: "ClusterRoleBinding", + metadata: { + name: "spartakus", + }, + roleRef: { + kind: "ClusterRole", + name: "spartakus", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "spartakus", + namespace: namespace, + }, + ], + }, // clusterRoleBinding + deployVolunteer(usageId): { + apiVersion: "apps/v1beta2", + kind: "Deployment", + metadata: { + labels: { + app: "pipeline-spartakus-volunteer", + }, + name: "pipeline-spartakus-volunteer", + namespace: namespace, + }, + spec: { + selector: { + matchLabels: { + app: "pipeline-spartakus-volunteer", + }, + }, + replicas: 1, + template: { + metadata: { + labels: { + app: "pipeline-spartakus-volunteer", + }, + }, + spec: { + containers: [ + { + name: "pipeline-spartakus-volunteer", + image: "gcr.io/google_containers/spartakus-amd64:v1.0.0", + imagePullPolicy: "IfNotPresent", + args: [ + "volunteer", + "--cluster-id=" + usageId, + "--database=https://ml-pipeline-reporting.appspot.com/", + ], + }, + ], + serviceAccountName: "spartakus", + }, + }, + }, + }, // deployVolunteer + }, // parts +}