feat(mlmd): Introduce PostgreSQL kustomization for MLMD. (#9927)

2023-08-24 10:16:16 -07:00 · 2023-08-24 10:16:16 -07:00 · b6be4ea79b
parent 9ce2866527
commit b6be4ea79b
17 changed files with 233 additions and 256 deletions
--- a/manifests/kustomize/base/cache/postgresql/kustomization.yaml
+++ b/manifests/kustomize/base/cache/postgresql/kustomization.yaml
@ -1,13 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - cache-deployment.yaml
-  - ../cache-role.yaml
-  - ../cache-rolebinding.yaml
-  - ../cache-sa.yaml
-  - ../cache-service.yaml
-commonLabels:
-  app: cache-server
-images:
-  - name: gcr.io/ml-pipeline/cache-server
-    newTag: 2.0.0
--- a/manifests/kustomize/base/installs/generic/postgres/kustomization.yaml
+++ b/manifests/kustomize/base/installs/generic/postgres/kustomization.yaml
@ -2,12 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: kubeflow
 bases:
- ../../../pipeline/postgres
- ../../../cache/postgres
+- ../../../postgresql/pipeline
+- ../../../postgresql/cache
 - ../../../cache-deployer
 resources:
 - pipeline-install-config.yaml
- postgres-secret.yaml
+- postgres-secret-extended.yaml
 vars:
 - name: kfp-namespace
  objref:
--- a/manifests/kustomize/base/installs/generic/postgres/postgres-secret-extended.yaml
+++ b/manifests/kustomize/base/installs/generic/postgres/postgres-secret-extended.yaml
@ -1,7 +1,7 @@
 kind: Secret
 apiVersion: v1
 metadata:
-  name: postgres-secret
+  name: postgres-secret-extended
 stringData:
  username: user
  password: "password"
--- a/manifests/kustomize/base/metadata/overlays/postgres/kustomization.yaml
+++ b/manifests/kustomize/base/metadata/overlays/postgres/kustomization.yaml
@ -0,0 +1,38 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+
+bases:
+- ../../base
+resources:
+- metadata-db-pvc.yaml
+- metadata-db-deployment.yaml
+- metadata-db-service.yaml
+
+patchesStrategicMerge:
+- patches/metadata-grpc-deployment.yaml
+
+configMapGenerator:
+- name: metadata-postgres-db-parameters
+  envs:
+  - params.env
+secretGenerator:
+- name: metadata-postgres-db-secrets
+  envs:
+  - secrets.env
+generatorOptions:
+  disableNameSuffixHash: true
+
+images:
+- name: postgres
+  newName: postgres
+  newTag: 14.7-alpine3.17
+
+vars:
+- name: MLMD_DB_HOST
+  objref:
+    kind: Service
+    name: metadata-postgres-db
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.name
--- a/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-deployment.yaml
+++ b/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-deployment.yaml
@ -0,0 +1,43 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metadata-postgres-db
+  labels:
+    component: db
+spec:
+  selector:
+    matchLabels:
+      component: db
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      name: db
+      labels:
+        component: db
+      annotations:
+        sidecar.istio.io/inject: "false"
+    spec:
+      containers:
+      - name: db-container
+        image: postgres
+        env:
+          - name: PGDATA
+            value: /var/lib/postgresql/data/pgdata
+        envFrom:
+        - configMapRef:
+            name: metadata-postgres-db-parameters
+        - secretRef:
+            name: metadata-postgres-db-secrets
+        ports:
+        - name: postgres
+          containerPort: 5432
+        volumeMounts:
+        - name: metadata-postgres
+          mountPath: /var/lib/postgresql/data
+      volumes:
+      - name: metadata-postgres
+        persistentVolumeClaim:
+          claimName: metadata-postgres
+
--- a/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-pvc.yaml
+++ b/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-pvc.yaml
@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: metadata-postgres
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi
--- a/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-service.yaml
+++ b/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-service.yaml
@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: metadata-postgres-db
+  labels:
+    component: db
+spec:
+  type: ClusterIP
+  ports:
+    - port: 5432
+      protocol: TCP
+      name: postgres
+  selector:
+    component: db
--- a/manifests/kustomize/base/metadata/overlays/postgres/params.env
+++ b/manifests/kustomize/base/metadata/overlays/postgres/params.env
@ -0,0 +1,2 @@
+POSTGRES_PORT=5432
+POSTGRES_DBNAME=mlmdpostgres
--- a/manifests/kustomize/base/metadata/overlays/postgres/patches/metadata-grpc-deployment.yaml
+++ b/manifests/kustomize/base/metadata/overlays/postgres/patches/metadata-grpc-deployment.yaml
@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metadata-grpc-deployment
+spec:
+  template:
+    spec:
+      containers:
+        - name: container
+          # Remove existing environment variables
+          env:
+          - $patch: replace
+          envFrom:
+          - configMapRef:
+              name: metadata-postgres-db-parameters
+          - secretRef:
+              name: metadata-postgres-db-secrets
+          - configMapRef:
+              name: metadata-grpc-configmap
+          args: ["--grpc_port=$(METADATA_GRPC_SERVICE_PORT)",
+                 "--metadata_source_config_type=postgresql",
+                 "--postgres_config_host=$(MLMD_DB_HOST)",
+                 "--postgres_config_port=$(POSTGRES_PORT)",
+                 "--postgres_config_dbname=$(POSTGRES_DBNAME)",
+                 "--postgres_config_user=$(POSTGRES_USER)",
+                 "--postgres_config_password=$(POSTGRES_PASSWORD)",
+                #  "--postgres_config_skip_db_creation=true",
+                 "--enable_database_upgrade=true"]
--- a/manifests/kustomize/base/metadata/overlays/postgres/secrets.env
+++ b/manifests/kustomize/base/metadata/overlays/postgres/secrets.env
@ -0,0 +1,2 @@
+POSTGRES_USER=root
+POSTGRES_PASSWORD=password
--- a/manifests/kustomize/base/pipeline/postgres/kustomization.yaml
+++ b/manifests/kustomize/base/pipeline/postgres/kustomization.yaml
@ -1,50 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-bases:
-  - ../metadata-writer
-resources:
-  - ml-pipeline-apiserver-deployment.yaml
-  - ../ml-pipeline-apiserver-role.yaml
-  - ../ml-pipeline-apiserver-rolebinding.yaml
-  - ../ml-pipeline-apiserver-sa.yaml
-  - ../ml-pipeline-apiserver-service.yaml
-  - ../ml-pipeline-persistenceagent-deployment.yaml
-  - ../ml-pipeline-persistenceagent-role.yaml
-  - ../ml-pipeline-persistenceagent-rolebinding.yaml
-  - ../ml-pipeline-persistenceagent-sa.yaml
-  - ../ml-pipeline-scheduledworkflow-deployment.yaml
-  - ../ml-pipeline-scheduledworkflow-role.yaml
-  - ../ml-pipeline-scheduledworkflow-rolebinding.yaml
-  - ../ml-pipeline-scheduledworkflow-sa.yaml
-  - ../ml-pipeline-ui-deployment.yaml
-  - ../ml-pipeline-ui-configmap.yaml
-  - ../ml-pipeline-ui-role.yaml
-  - ../ml-pipeline-ui-rolebinding.yaml
-  - ../ml-pipeline-ui-sa.yaml
-  - ../ml-pipeline-ui-service.yaml
-  - ../ml-pipeline-viewer-crd-role.yaml
-  - ../ml-pipeline-viewer-crd-rolebinding.yaml
-  - ../ml-pipeline-viewer-crd-deployment.yaml
-  - ../ml-pipeline-viewer-crd-sa.yaml
-  - ../ml-pipeline-visualization-deployment.yaml
-  - ../ml-pipeline-visualization-sa.yaml
-  - ../ml-pipeline-visualization-service.yaml
-  - ../pipeline-runner-role.yaml
-  - ../pipeline-runner-rolebinding.yaml
-  - ../pipeline-runner-sa.yaml
-  - ../container-builder-sa.yaml
-  - ../viewer-sa.yaml
-  - ../kfp-launcher-configmap.yaml
-images:
-  - name: gcr.io/ml-pipeline/api-server
-    newTag: 2.0.0
-  - name: gcr.io/ml-pipeline/persistenceagent
-    newTag: 2.0.0
-  - name: gcr.io/ml-pipeline/scheduledworkflow
-    newTag: 2.0.0
-  - name: gcr.io/ml-pipeline/frontend
-    newTag: 2.0.0
-  - name: gcr.io/ml-pipeline/viewer-crd-controller
-    newTag: 2.0.0
-  - name: gcr.io/ml-pipeline/visualization-server
-    newTag: 2.0.0
--- a/manifests/kustomize/base/pipeline/postgres/ml-pipeline-apiserver-deployment.yaml
+++ b/manifests/kustomize/base/pipeline/postgres/ml-pipeline-apiserver-deployment.yaml
@ -1,161 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: ml-pipeline
-  name: ml-pipeline
-spec:
-  selector:
-    matchLabels:
-      app: ml-pipeline
-  template:
-    metadata:
-      labels:
-        app: ml-pipeline
-      annotations:
-        cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
-    spec:
-      containers:
-      - env:
-        - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: autoUpdatePipelineDefaultVersion
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: OBJECTSTORECONFIG_SECURE
-          value: "false"
-        - name: OBJECTSTORECONFIG_BUCKETNAME
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: bucketName
-        # relic variables
-        - name: DBCONFIG_USER
-          valueFrom:
-            secretKeyRef:
-              name: mysql-secret
-              key: username
-        - name: DBCONFIG_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: mysql-secret
-              key: password
-        - name: DBCONFIG_DBNAME
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: pipelineDb
-        - name: DBCONFIG_HOST
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: dbHost
-        - name: DBCONFIG_PORT
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: dbPort
-        # end of relic variables
-        - name: DBCONFIG_CONMAXLIFETIME
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: ConMaxLifeTime
-        - name: DB_DRIVER_NAME
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: dbType
-        # PostgreSQL Config
-        - name: DBCONFIG_POSTGRESQLCONFIG_USER
-          valueFrom:
-            secretKeyRef:
-              name: postgres-secret
-              key: username
-        - name: DBCONFIG_POSTGRESQLCONFIG_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: postgres-secret
-              key: password
-        - name: DBCONFIG_POSTGRESQLCONFIG_DBNAME
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: pipelineDb
-        - name: DBCONFIG_POSTGRESQLCONFIG_HOST
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: postgresHost
-        - name: DBCONFIG_POSTGRESQLCONFIG_PORT
-          valueFrom:
-            configMapKeyRef:
-              name: pipeline-install-config
-              key: postgresPort
-        # end of PostgreSQL variables
-        - name: OBJECTSTORECONFIG_ACCESSKEY
-          valueFrom:
-            secretKeyRef:
-              name: mlpipeline-minio-artifact
-              key: accesskey
-        - name: OBJECTSTORECONFIG_SECRETACCESSKEY
-          valueFrom:
-            secretKeyRef:
-              name: mlpipeline-minio-artifact
-              key: secretkey
-        image: gcr.io/ml-pipeline/api-server:dummy
-        imagePullPolicy: IfNotPresent
-        name: ml-pipeline-api-server
-        ports:
-        - name: http
-          containerPort: 8888
-        - name: grpc
-          containerPort: 8887
-        readinessProbe:
-          exec:
-            command:
-              - wget
-              - -q # quiet
-              - -S # show server response
-              - -O
-              - "-" # Redirect output to stdout
-              - http://localhost:8888/apis/v1beta1/healthz
-          initialDelaySeconds: 3
-          periodSeconds: 5
-          timeoutSeconds: 2
-        livenessProbe:
-          exec:
-            command:
-              - wget
-              - -q # quiet
-              - -S # show server response
-              - -O
-              - "-" # Redirect output to stdout
-              - http://localhost:8888/apis/v1beta1/healthz
-          initialDelaySeconds: 3
-          periodSeconds: 5
-          timeoutSeconds: 2
-        # This startup probe provides up to a 60 second grace window before the
-        # liveness probe takes over to accomodate the occasional database
-        # migration.
-        startupProbe:
-          exec:
-            command:
-              - wget
-              - -q # quiet
-              - -S # show server response
-              - -O
-              - "-" # Redirect output to stdout
-              - http://localhost:8888/apis/v1beta1/healthz
-          failureThreshold: 12
-          periodSeconds: 5
-          timeoutSeconds: 2
-        resources:
-          requests:
-            cpu: 250m
-            memory: 500Mi
-      serviceAccountName: ml-pipeline
--- a/manifests/kustomize/base/postgresql/cache/cache-deployment-patch.yaml
+++ b/manifests/kustomize/base/postgresql/cache/cache-deployment-patch.yaml
@ -2,22 +2,13 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: cache-server
-  labels:
-    app: cache-server
 spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: cache-server
  template:
-    metadata:
-      labels:
-        app: cache-server
    spec:
      containers:
      - name: server
-        image: gcr.io/ml-pipeline/cache-server:dummy
        env:
+          - $patch: replace
          - name: DEFAULT_CACHE_STALENESS
            valueFrom:
              configMapKeyRef:
@ -49,21 +40,21 @@ spec:
            valueFrom:
              configMapKeyRef:
                name: pipeline-install-config
-                key: postgresDbHost
+                key: postgresHost
          - name: DBCONFIG_PORT
            valueFrom:
              configMapKeyRef:
                name: pipeline-install-config
-                key: postgresDbPort
+                key: postgresPort
          - name: DBCONFIG_USER
            valueFrom:
              secretKeyRef:
-                name: postgres-secret
+                name: postgres-secret-extended
                key: username
          - name: DBCONFIG_PASSWORD
            valueFrom:
              secretKeyRef:
-                name: postgres-secret
+                name: postgres-secret-extended
                key: password
          - name: NAMESPACE_TO_WATCH
            valueFrom:
@ -82,16 +73,3 @@ spec:
               "--namespace_to_watch=$(NAMESPACE_TO_WATCH)",
               "--listen_port=$(WEBHOOK_PORT)",
              ]
-        imagePullPolicy: Always
-        ports:
-        - containerPort: 8443
-          name: webhook-api
-        volumeMounts:
-        - name: webhook-tls-certs
-          mountPath: /etc/webhook/certs
-          readOnly: true
-      volumes:
-      - name: webhook-tls-certs
-        secret:
-          secretName: webhook-server-tls
-      serviceAccountName: kubeflow-pipelines-cache
--- a/manifests/kustomize/base/postgresql/cache/kustomization.yaml
+++ b/manifests/kustomize/base/postgresql/cache/kustomization.yaml
@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../../cache
+patchesStrategicMerge:
+- cache-deployment-patch.yaml
--- a/manifests/kustomize/base/postgresql/pipeline/kustomization.yaml
+++ b/manifests/kustomize/base/postgresql/pipeline/kustomization.yaml
@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+  - ../../pipeline
+patchesStrategicMerge:
+  - ml-pipeline-apiserver-deployment-patch.yaml
--- a/manifests/kustomize/base/postgresql/pipeline/ml-pipeline-apiserver-deployment-patch.yaml
+++ b/manifests/kustomize/base/postgresql/pipeline/ml-pipeline-apiserver-deployment-patch.yaml
@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+      - name: ml-pipeline-api-server
+        env:
+        - $patch: replace
+        - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: autoUpdatePipelineDefaultVersion
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: OBJECTSTORECONFIG_SECURE
+          value: "false"
+        - name: OBJECTSTORECONFIG_BUCKETNAME
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: bucketName
+        - name: DBCONFIG_CONMAXLIFETIME
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: ConMaxLifeTime
+        - name: DB_DRIVER_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: dbType
+        # PostgreSQL Config
+        - name: DBCONFIG_POSTGRESQLCONFIG_USER
+          valueFrom:
+            secretKeyRef:
+              name: postgres-secret-extended
+              key: username
+        - name: DBCONFIG_POSTGRESQLCONFIG_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: postgres-secret-extended
+              key: password
+        - name: DBCONFIG_POSTGRESQLCONFIG_DBNAME
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: pipelineDb
+        - name: DBCONFIG_POSTGRESQLCONFIG_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: postgresHost
+        - name: DBCONFIG_POSTGRESQLCONFIG_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: postgresPort
+        # end of PostgreSQL variables
+        - name: OBJECTSTORECONFIG_ACCESSKEY
+          valueFrom:
+            secretKeyRef:
+              name: mlpipeline-minio-artifact
+              key: accesskey
+        - name: OBJECTSTORECONFIG_SECRETACCESSKEY
+          valueFrom:
+            secretKeyRef:
+              name: mlpipeline-minio-artifact
+              key: secretkey
--- a/manifests/kustomize/env/platform-agnostic-postgresql/kustomization.yaml
+++ b/manifests/kustomize/env/platform-agnostic-postgresql/kustomization.yaml
@ -3,7 +3,7 @@ kind: Kustomization

 bases:
  - ../../base/installs/generic/postgres
-  - ../../base/metadata/base
+  - ../../base/metadata/overlays/postgres
  - ../../third-party/argo/installs/namespace
  - ../../third-party/minio/base
  - ../../third-party/postgresql/base