Add sidecar container support for linkerd-prometheus helm chart (#4761)

* Add sidecar container support for linkerd-prometheus

Adds a new setting to the Prometheus' Helm config, allowing adding any kind of sidecar containers to the main container.

The specific use case that inspired this was for exporting data from Prometheus to external systems (e.g. cloudwatch, stackdriver, datadog) using a process that watches the prometheus write-ahead log (WAL).

Signed-off-by: Nathan J. Mehl <n@oden.io>

charts/add-ons/prometheus/templates/prometheus.yaml

@@ -231,6 +231,9 @@ spec:
       securityContext:
         fsGroup: 65534
       containers:
+      {{- if .Values.sidecarContainers -}}
+      {{- toYaml .Values.sidecarContainers | trim | nindent 6 }}
+      {{- end}}
       - args:
         {{- range $key, $value := .Values.args}}
         - --{{ $key }}{{ if $value }}={{ $value }}{{ end }}

charts/linkerd2/README.md

@@ -229,6 +229,7 @@ The following table lists the configurable parameters for the Prometheus Add-On.
 | `prometheus.resources.memory.request`  | Amount of memory that the prometheus container requests                                                                                                                               ||
 | `prometheus.ruleConfigMapMounts`             | Alerting/recording rule ConfigMap mounts (sub-path names must end in `_rules.yml` or `_rules.yaml`)                                                                                   | `[]`                                 |
 | `prometheus.scrapeConfigs`             | A scrape_config section specifies a set of targets and parameters describing how to scrape them.                                                        | `[]`                                 |
+| `prometheus.sidecarContainers`         | A sidecarContainers section specifies a list of secondary containers to run in the prometheus pod e.g. to export data to non-prometheus systems | `[]`                                 |
 
 Most of the above configuration match directly with the official Prometheus
 configuration which can be found [here](https://prometheus.io/docs/prometheus/latest/configuration/configuration)

charts/linkerd2/values.yaml

@@ -298,6 +298,30 @@ prometheus:
   # - name: recording-rules
   #   subPath: recording_rules.yml
   #   configMap: linkerd-prometheus-rules
+  ###
+  ### Sidecar containers allow access to the prometheus data directory,
+  ### e.g. for exporting data to non-prometheus systems.
+  # sidecarContainers:
+  # - name: sidecar
+  #   image: gcr.io/myproject/stackdriver-prometheus-sidecar
+  #   imagePullPolicy: Always
+  #   command:
+  #   - /bin/sh
+  #   - -c
+  #   - |
+  #     exec /bin/stackdriver-prometheus-sidecar \
+  #       --stackdriver.project-id=myproject \
+  #       --stackdriver.kubernetes.location=us-central1 \
+  #       --stackdriver.kubernetes.cluster-name=mycluster \
+  #       --prometheus.wal-directory=/data/wal \
+  #       --log.level=info
+  #   volumeMounts:
+  #   - mountPath: /data
+  #     name: data
+  #   ports:
+  #   - name: foo
+  #     containerPort: 9091
+  #     protocol: TCP
   ### WARNING: persistence is experimental and has not been tested/vetted by the Linkerd team.
   ### As such, please refer to https://linkerd.io/2/tasks/exporting-metrics/ for the recommended approach to metrics data retention.
   # if enabled, creates a persistent volume claim for prometheus data

cli/cmd/testdata/install_prometheus_overwrite.golden

@@ -2555,6 +2555,28 @@ data:
         scheme: https
         tls_config:
           ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      sidecarContainers:
+      - command:
+        - /bin/sh
+        - -c
+        - |
+          exec /bin/stackdriver-prometheus-sidecar \
+            --stackdriver.project-id=myproject \
+            --stackdriver.kubernetes.location=us-central1 \
+            --stackdriver.kubernetes.cluster-name=mycluster \
+            --prometheus.wal-directory=/data/wal \
+            --log.level=info
+          volumeMounts:
+          - mountPath: /data
+            name: data
+        imagePullPolicy: always
+        lifecycle:
+          type: Sidecar
+        name: sidecar
+        ports:
+        - containerPort: 9091
+          name: foo
+          protocol: TCP
     tracing:
       enabled: false
 ---
@@ -3139,6 +3161,27 @@ spec:
       securityContext:
         fsGroup: 65534
       containers:
+      - command:
+        - /bin/sh
+        - -c
+        - |
+          exec /bin/stackdriver-prometheus-sidecar \
+            --stackdriver.project-id=myproject \
+            --stackdriver.kubernetes.location=us-central1 \
+            --stackdriver.kubernetes.cluster-name=mycluster \
+            --prometheus.wal-directory=/data/wal \
+            --log.level=info
+          volumeMounts:
+          - mountPath: /data
+            name: data
+        imagePullPolicy: always
+        lifecycle:
+          type: Sidecar
+        name: sidecar
+        ports:
+        - containerPort: 9091
+          name: foo
+          protocol: TCP
       - args:
         - --config.file=/etc/prometheus/prometheus.yml
         - --log.format=json

cli/cmd/testdata/prom-config.yaml

@@ -35,3 +35,25 @@ prometheus:
     configMap: linkerd-prometheus-rules
   remoteWrite:
   - url: http://cortex-service.default:9009/api/prom/push
+  sidecarContainers:
+  - name: sidecar
+    lifecycle:
+      type: Sidecar
+    imagePullPolicy: always
+    command:
+    - /bin/sh
+    - -c
+    - |
+      exec /bin/stackdriver-prometheus-sidecar \
+        --stackdriver.project-id=myproject \
+        --stackdriver.kubernetes.location=us-central1 \
+        --stackdriver.kubernetes.cluster-name=mycluster \
+        --prometheus.wal-directory=/data/wal \
+        --log.level=info
+      volumeMounts:
+      - mountPath: /data
+        name: data
+    ports:
+    - name: foo
+      containerPort: 9091
+      protocol: TCP