Include viz components in Prom scrapes, fix Linkerd Health charts (#5656)

* Include viz components in Prom scrapes, fix Linkerd Health charts

Fixes #5429

Expanded the `linkerd-controller` Prometheus scraping config so it also includes the `linkerd-viz` namespace. Also simplified the first relabelling config there removing the `_meta_kubernetes_pod_label_linkerd_io_control_plane_component` source label that wasn't serving any purpose. Just by its own, that extra scraping now allows having non-empty Go charts at the bottom of the `Linkerd Health` charts for the viz components.

Additionally, the `namespace-viz` variable was added into `health.json` which then is leveraged in the queries for the `Control-Plane Traffic` and `Control-Plane TCP Metrics` charts to include the viz pods.

Finally in that same file the queries for the `Data-Plane Telemetry` section were simplified by removing the filter on the `control_plane_ns` label which was redundant.

grafana/dashboards/health.json

@@ -73,14 +73,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "process_virtual_memory_bytes{control_plane_ns=\"$namespace\", job=\"linkerd-proxy\"}",
+          "expr": "process_virtual_memory_bytes{job=\"linkerd-proxy\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{namespace}}/{{pod}}/virtual",
           "refId": "A"
         },
         {
-          "expr": "process_resident_memory_bytes{control_plane_ns=\"$namespace\", job=\"linkerd-proxy\"}",
+          "expr": "process_resident_memory_bytes{job=\"linkerd-proxy\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{namespace}}/{{pod}}/resident",
@@ -166,7 +166,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(process_cpu_seconds_total{control_plane_ns=\"$namespace\", job=\"linkerd-proxy\"}[20s])) by (namespace, pod)",
+          "expr": "sum(irate(process_cpu_seconds_total{job=\"linkerd-proxy\"}[20s])) by (namespace, pod)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{namespace}}/{{pod}}",
@@ -252,7 +252,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "process_open_fds{control_plane_ns=\"$namespace\", job=\"linkerd-proxy\"}",
+          "expr": "process_open_fds{job=\"linkerd-proxy\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{namespace}}/{{pod}}",
@@ -359,7 +359,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(response_total{classification=\"success\", deployment!=\"\", namespace=\"$namespace\", direction=\"inbound\"}[30s])) by (pod) / sum(irate(response_total{deployment!=\"\", namespace=\"$namespace\", direction=\"inbound\"}[30s])) by (pod)",
+          "expr": "sum(irate(response_total{classification=\"success\", deployment!=\"\", namespace=~\"$namespace|$namespace-viz\", direction=\"inbound\"}[30s])) by (pod) / sum(irate(response_total{deployment!=\"\", namespace=~\"$namespace|$namespace-viz\", direction=\"inbound\"}[30s])) by (pod)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "po/{{pod}}",
@@ -450,7 +450,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(request_total{deployment!=\"\", namespace=\"$namespace\", direction=\"inbound\"}[30s])) by (pod)",
+          "expr": "sum(irate(request_total{deployment!=\"\", namespace=~\"$namespace|$namespace-viz\", direction=\"inbound\"}[30s])) by (pod)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "po/{{pod}}",
@@ -542,7 +542,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum(rate(response_latency_ms_bucket{deployment!=\"\", namespace=\"$namespace\", direction=\"inbound\"}[30s])) by (le, pod))",
+          "expr": "histogram_quantile(0.95, sum(rate(response_latency_ms_bucket{deployment!=\"\", namespace=~\"$namespace|$namespace-viz\", direction=\"inbound\"}[30s])) by (le, pod))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "po/{{pod}}",
@@ -681,7 +681,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "tcp_close_total{deployment=\"$deployment\", namespace=\"$namespace\", direction=\"inbound\",errno!=\"\"}",
+              "expr": "tcp_close_total{deployment=\"$deployment\", namespace=~\"$namespace|$namespace-viz\", direction=\"inbound\",errno!=\"\"}",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "{{peer}} {{errno}}",
@@ -772,7 +772,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "tcp_open_connections{deployment=\"$deployment\", namespace=\"$namespace\", direction=\"inbound\"}",
+              "expr": "tcp_open_connections{deployment=\"$deployment\", namespace=~\"$namespace|$namespace-viz\", direction=\"inbound\"}",
               "format": "time_series",
               "intervalFactor": 1,
               "legendFormat": "{{peer}}",
@@ -857,7 +857,7 @@
           },
           "targets": [
             {
-              "expr": "tcp_connection_duration_ms_bucket{deployment=\"$deployment\", namespace=\"$namespace\", direction=\"inbound\"}",
+              "expr": "tcp_connection_duration_ms_bucket{deployment=\"$deployment\", namespace=~\"$namespace|$namespace-viz\", direction=\"inbound\"}",
               "format": "time_series",
               "intervalFactor": 1,
               "refId": "A"
@@ -2246,7 +2246,7 @@
         "multi": false,
         "name": "deployment",
         "options": [],
-        "query": "label_values(process_start_time_seconds{control_plane_component!=\"\"}, deployment)",
+        "query": "label_values(process_start_time_seconds{namespace=~\"$namespace|$namespace-viz\"}, deployment)",
         "refresh": 2,
         "regex": "",
         "skipUrlSync": false,
@@ -2303,6 +2303,28 @@
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {},
+        "datasource": "prometheus",
+        "definition": "",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Linkerd Viz Namespace",
+        "multi": false,
+        "name": "namespace-viz",
+        "options": [],
+        "query": "label_values(process_start_time_seconds{control_plane_ns!=\"\",extension!=\"\"},  namespace)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
       }
     ]
   },

viz/charts/linkerd-viz/templates/prometheus.yaml

@@ -73,13 +73,14 @@ data:
       kubernetes_sd_configs:
       - role: pod
         namespaces:
-          names: ['{{.Values.linkerdNamespace}}']
+          names:
+          - '{{.Values.linkerdNamespace}}'
+          - '{{.Values.namespace}}'
       relabel_configs:
       - source_labels:
-        - __meta_kubernetes_pod_label_linkerd_io_control_plane_component
         - __meta_kubernetes_pod_container_port_name
         action: keep
-        regex: (.*);admin-http$
+        regex: admin-http
       - source_labels: [__meta_kubernetes_pod_container_name]
         action: replace
         target_label: component

viz/cmd/testdata/install_default.golden

@@ -693,13 +693,14 @@ data:
       kubernetes_sd_configs:
       - role: pod
         namespaces:
-          names: ['linkerd']
+          names:
+          - 'linkerd'
+          - 'linkerd-viz'
       relabel_configs:
       - source_labels:
-        - __meta_kubernetes_pod_label_linkerd_io_control_plane_component
         - __meta_kubernetes_pod_container_port_name
         action: keep
-        regex: (.*);admin-http$
+        regex: admin-http
       - source_labels: [__meta_kubernetes_pod_container_name]
         action: replace
         target_label: component

viz/cmd/testdata/install_proxy_resources.golden

@@ -697,13 +697,14 @@ data:
       kubernetes_sd_configs:
       - role: pod
         namespaces:
-          names: ['linkerd']
+          names:
+          - 'linkerd'
+          - 'linkerd-viz'
       relabel_configs:
       - source_labels:
-        - __meta_kubernetes_pod_label_linkerd_io_control_plane_component
         - __meta_kubernetes_pod_container_port_name
         action: keep
-        regex: (.*);admin-http$
+        regex: admin-http
       - source_labels: [__meta_kubernetes_pod_container_name]
         action: replace
         target_label: component