Add readiness/liveness checks for third party components (#1121)

* Add readiness/liveness checks for third party components

Any possible issues with the third party control plane components can wedge the services.

Take the best practices for prometheus/grafana and add them to our template. See #1116

* Update test fixtures for new output

cli/cmd/testdata/install_default.golden

@@ -411,10 +411,22 @@ spec:
         - --config.file=/etc/prometheus/prometheus.yml
         image: prom/prometheus:v2.2.1
         imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /-/healthy
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
         name: prometheus
         ports:
         - containerPort: 9090
           name: admin-http
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
         resources: {}
         volumeMounts:
         - mountPath: /etc/prometheus
@@ -635,10 +647,22 @@ spec:
       containers:
       - image: gcr.io/runconduit/grafana:undefined
         imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
         name: grafana
         ports:
         - containerPort: 3000
           name: http
+        readinessProbe:
+          failureThreshold: 10
+          httpGet:
+            path: /api/health
+            port: 3000
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          timeoutSeconds: 30
         resources: {}
         volumeMounts:
         - mountPath: /etc/grafana

cli/cmd/testdata/install_output.golden

@@ -414,10 +414,22 @@ spec:
         - --config.file=/etc/prometheus/prometheus.yml
         image: PrometheusImage
         imagePullPolicy: ImagePullPolicy
+        livenessProbe:
+          httpGet:
+            path: /-/healthy
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
         name: prometheus
         ports:
         - containerPort: 9090
           name: admin-http
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
         resources: {}
         volumeMounts:
         - mountPath: /etc/prometheus
@@ -639,10 +651,22 @@ spec:
       containers:
       - image: GrafanaImage
         imagePullPolicy: ImagePullPolicy
+        livenessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
         name: grafana
         ports:
         - containerPort: 3000
           name: http
+        readinessProbe:
+          failureThreshold: 10
+          httpGet:
+            path: /api/health
+            port: 3000
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          timeoutSeconds: 30
         resources: {}
         volumeMounts:
         - mountPath: /etc/grafana

cli/install/template.go

@@ -308,6 +308,18 @@ spec:
         args:
         - "--storage.tsdb.retention=6h"
         - "--config.file=/etc/prometheus/prometheus.yml"
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
+        livenessProbe:
+          httpGet:
+            path: /-/healthy
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
 
 ---
 kind: ConfigMap
@@ -482,6 +494,18 @@ spec:
           readOnly: true
         image: {{.GrafanaImage}}
         imagePullPolicy: {{.ImagePullPolicy}}
+        livenessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
+        readinessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
+          initialDelaySeconds: 60
+          timeoutSeconds: 30
+          failureThreshold: 10
+          periodSeconds: 10
 
 ---
 kind: ConfigMap