[Manifest] Cache - MKP deployment (#3430)

* Initial execution cache

This commit adds initial execution cache service. Including http service
and execution key generation.

* fix master

* Add cache manifests for mkp deployment

* revert go.sum

* Add helm on delete policy for cache deployer job

* Change cache deployer job to statefulset

* remove unnecessary cluster role

* seperate clusterrole and role

* add role and rolebinding to mkp

* change secret role to clusterrole

* Add cloudsql support to cache

* fix comma

* Change cache secret clusterrole to role

* Adjust sequences of resources

* Update values and schema

* remove extra tab

* Change statefulset to job

* Add pod delete permission to cache deployer role

* Test changing cache deployer job to deployment

* remove extra permission

* remove statefulset check

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/cache.yaml

@@ -0,0 +1,280 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-cache-deployer-sa
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-cache
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: kubeflow-pipelines-cache-deployer-clusterrole
+    app.kubernetes.io/name: {{ .Release.Name }}
+  name: kubeflow-pipelines-cache-deployer-clusterrole
+rules:
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  - certificatesigningrequests/approval
+  verbs:
+  - create
+  - delete
+  - get
+  - update
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  verbs:
+  - create
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: kubeflow-pipelines-cache-deployer-role
+    app.kubernetes.io/name: {{ .Release.Name }}
+  name: kubeflow-pipelines-cache-deployer-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - get
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: kubeflow-pipelines-cache-role
+    app.kubernetes.io/name: {{ .Release.Name }}
+  name: kubeflow-pipelines-cache-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-binding
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubeflow-pipelines-cache-role
+subjects:
+- kind: ServiceAccount
+  name: kubeflow-pipelines-cache
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-deployer-clusterrolebinding
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubeflow-pipelines-cache-deployer-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: kubeflow-pipelines-cache-deployer-sa
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-deployer-rolebinding
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubeflow-pipelines-cache-deployer-role
+subjects:
+- kind: ServiceAccount
+  name: kubeflow-pipelines-cache-deployer-sa
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cache-deployer-deployment
+  labels:
+    app: cache-deployer
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cache-deployer
+      app.kubernetes.io/name: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: cache-deployer
+        app.kubernetes.io/name: {{ .Release.Name }}
+    spec:
+      containers:
+      - name: main
+        image: {{ .Values.images.cachedeployer }}
+        imagePullPolicy: Always
+        env:
+        - name: NAMESPACE_TO_WATCH
+          value: {{ .Release.Namespace }}
+      serviceAccountName: kubeflow-pipelines-cache-deployer-sa
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cache-configmap
+  labels:
+    component: cache-server
+data:
+  {{ if .Values.managedstorage.databaseNamePrefix }}
+  mysql_database: '{{ .Values.managedstorage.databaseNamePrefix }}_cachedb'
+  {{ else }}
+  mysql_database: '{{ .Release.Name | replace "-" "_" | replace "." "_"}}_cachedb'
+  {{ end }}
+  mysql_driver: "mysql"
+  mysql_host: "mysql"
+  mysql_port: "3306" 
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cache-server
+  labels:
+    app: cache-server
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cache-server
+      app.kubernetes.io/name: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: cache-server
+        app.kubernetes.io/name: {{ .Release.Name }}
+    spec:
+      containers:
+      - name: server
+        image: {{ .Values.images.cacheserver }}
+        env:
+          {{ if .Values.managedstorage.enabled }}
+          - name: DBCONFIG_USER
+            valueFrom:
+              secretKeyRef:
+                name: mysql-credential
+                key: username
+          - name: DBCONFIG_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mysql-credential
+                key: password
+          {{ else }}
+          - name: DBCONFIG_USER
+            value: 'root'
+          - name: DBCONFIG_PASSWORD
+            value: ''
+          {{ end }}
+          - name: DBCONFIG_DRIVER
+            valueFrom:
+              configMapKeyRef:
+                name: cache-configmap
+                key: mysql_driver
+          - name: DBCONFIG_DB_NAME
+            valueFrom:
+              configMapKeyRef:
+                name: cache-configmap
+                key: mysql_database
+          - name: DBCONFIG_HOST_NAME
+            valueFrom:
+              configMapKeyRef:
+                name: cache-configmap
+                key: mysql_host
+          - name: DBCONFIG_PORT
+            valueFrom:
+              configMapKeyRef:
+                name: cache-configmap
+                key: mysql_port
+          - name: NAMESPACE_TO_WATCH
+            value: {{ .Release.Namespace }}
+        args: ["--db_driver=$(DBCONFIG_DRIVER)",
+               "--db_host=$(DBCONFIG_HOST_NAME)",
+               "--db_port=$(DBCONFIG_PORT)",
+               "--db_name=$(DBCONFIG_DB_NAME)",
+               "--db_user=$(DBCONFIG_USER)",
+               "--db_password=$(DBCONFIG_PASSWORD)",
+               "--namespace_to_watch=$(NAMESPACE_TO_WATCH)",
+               ]
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8443
+          name: webhook-api
+        volumeMounts:
+        - name: webhook-tls-certs
+          mountPath: /etc/webhook/certs
+          readOnly: true
+      volumes:
+      - name: webhook-tls-certs
+        secret:
+          secretName: webhook-server-tls
+      serviceAccountName: kubeflow-pipelines-cache
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: cache-server
+  labels: 
+    app: cache-server
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  selector:
+    app: cache-server
+    app.kubernetes.io/name: {{ .Release.Name }}
+  ports:
+    - port: 443
+      targetPort: webhook-api

manifests/gcp_marketplace/chart/kubeflow-pipelines/values.yaml

@@ -14,6 +14,8 @@ images:
   visualizationserver: gcr.io/ml-pipeline/google/pipelines/visualizationserver:dummy
   metadataenvoy: gcr.io/ml-pipeline/google/pipelines/metadataenvoy:dummy
   metadatawriter: gcr.io/ml-pipeline/google/pipelines/metadatawriter:dummy
+  cacheserver: gcr.io/ml-pipeline/google/pipelines/cacheserver:dummy
+  cachedeployer: gcr.io/ml-pipeline/google/pipelines/cachedeployer:dummy
 
 gcpSecretName: "user-gcp-sa"
 serviceAccountCredential: ""

manifests/gcp_marketplace/schema.yaml

@@ -77,13 +77,21 @@ x-google-marketplace:
       properties:
         images.metadatawriter:
           type: FULL
+    cacheserver:
+      properties:
+        images.cacheserver:
+          type: FULL
+    cachedeployer:
+      properties:
+        images.cachedeployer:
+          type: FULL
   deployerServiceAccount:
     roles:
       - type: ClusterRole        # This is a cluster-wide ClusterRole
         rulesType: CUSTOM        # We specify our own custom RBAC roles
         rules:
-          - apiGroups: ['apiextensions.k8s.io']
-            resources: ['customresourcedefinitions']
+          - apiGroups: ['apiextensions.k8s.io', 'rbac.authorization.k8s.io']
+            resources: ['customresourcedefinitions', 'clusterroles', 'clusterrolebindings']
             verbs: ['*']
   clusterConstraints:
     resources:

manifests/kustomize/base/cache-deployer/cache-deployer-deployment.yaml

@@ -1,12 +1,11 @@
 apiVersion: apps/v1
-kind: StatefulSet
+kind: Deployment
 metadata:
-  name: cache-deployer-statefulset
+  name: cache-deployer-deployment
   labels:
     app: cache-deployer
 spec:
   replicas: 1
-  serviceName: cache-deployer
   selector:
     matchLabels:
       app: cache-deployer
@@ -25,5 +24,4 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
       serviceAccountName: kubeflow-pipelines-cache-deployer-sa
-      restartPolicy: Always
-  volumeClaimTemplates: []
+      restartPolicy: Always

manifests/kustomize/base/cache-deployer/cache-deployer-role.yaml

@@ -2,8 +2,8 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   labels:
-    app: kubeflow-pipelines-cache-deployer-secret-role
-  name: kubeflow-pipelines-cache-deployer-secret-role
+    app: kubeflow-pipelines-cache-deployer-role
+  name: kubeflow-pipelines-cache-deployer-role
 rules:
 - apiGroups:
   - ""

manifests/kustomize/base/cache-deployer/cache-deployer-rolebinding.yaml

@@ -5,7 +5,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: kubeflow-pipelines-cache-deployer-secret-role
+  name: kubeflow-pipelines-cache-deployer-role
 subjects:
 - kind: ServiceAccount
   name: kubeflow-pipelines-cache-deployer-sa

manifests/kustomize/base/cache-deployer/kustomization.yaml

@@ -2,8 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - cache-deployer-secret-role.yaml
+  - cache-deployer-role.yaml
   - cache-deployer-rolebinding.yaml
   - cache-deployer-sa.yaml
-  - cache-deployer-statefulset.yaml
+  - cache-deployer-deployment.yaml
   

test/deploy-pipeline-lite.sh

@@ -92,17 +92,12 @@ echo "Status of pods after kubectl apply"
 kubectl get pods -n ${NAMESPACE}
 
 # wait for all deployments to be successful
-# note, after we introduce daemonsets, we need to wait their rollout status here too
+# note, after we introduce statefulset and daemonsets, we need to wait their rollout status here too
 for deployment in $(kubectl get deployments -n ${NAMESPACE} -o name)
 do
   kubectl rollout status $deployment -n ${NAMESPACE}
 done
 
-for statefulset in $(kubectl get statefulset -n ${NAMESPACE} -o name)
-do
-  kubectl rollout status $statefulset -n ${NAMESPACE}
-done
-
 echo "Status of pods after rollouts are successful"
 kubectl get pods -n ${NAMESPACE}