prometheus: add add-on checks (#4756)

As linkerd-prometheus is optional now, the checks are also separated
and should only work when the prometheus add-on is installed.

This is done by re-using the add-on check code.

pkg/charts/linkerd2/prometheus.go

@@ -6,15 +6,16 @@ import (
 )
 
 var (
-	prometheusAddOn = "prometheus"
+	// PrometheusAddOn is the name of the prometheus add-on
+	PrometheusAddOn = "prometheus"
 )
 
 // Prometheus is an add-on that installs the prometheus component
 type Prometheus map[string]interface{}
 
-// Name returns the name of the Tracing add-on
+// Name returns the name of the Prometheus add-on
 func (p Prometheus) Name() string {
-	return prometheusAddOn
+	return PrometheusAddOn
 }
 
 // Values returns the configuration values that were assigned for this add-on

pkg/healthcheck/healthcheck_addons.go

@@ -18,13 +18,16 @@ const (
 	// LinkerdGrafanaAddOnChecks adds checks related to grafana add-on components
 	LinkerdGrafanaAddOnChecks CategoryID = "linkerd-grafana"
 
+	// LinkerdPrometheusAddOnChecks adds checks related to Prometheus add-on components
+	LinkerdPrometheusAddOnChecks CategoryID = "linkerd-prometheus"
+
 	// LinkerdTracingAddOnChecks adds checks related to tracing add-on components
 	LinkerdTracingAddOnChecks CategoryID = "linkerd-tracing"
 )
 
 var (
 	// AddOnCategories is the list of add-on category checks
-	AddOnCategories = []CategoryID{LinkerdAddOnChecks, LinkerdGrafanaAddOnChecks, LinkerdTracingAddOnChecks}
+	AddOnCategories = []CategoryID{LinkerdAddOnChecks, LinkerdPrometheusAddOnChecks, LinkerdGrafanaAddOnChecks, LinkerdTracingAddOnChecks}
 )
 
 // addOnCategories contain all the checks w.r.t add-ons. It is strongly advised to
@@ -44,6 +47,51 @@ func (hc *HealthChecker) addOnCategories() []category {
 				},
 			},
 		},
+		{
+			id: LinkerdPrometheusAddOnChecks,
+			checkers: []checker{
+				{
+					description: "prometheus add-on service account exists",
+					warning:     true,
+					check: func(context.Context) error {
+						if _, ok := hc.addOns[l5dcharts.PrometheusAddOn]; ok {
+							return hc.checkServiceAccounts([]string{"linkerd-prometheus"}, hc.ControlPlaneNamespace, "")
+						}
+						return &SkipError{Reason: "prometheus add-on not enabled"}
+					},
+				},
+				{
+					description: "prometheus add-on config map exists",
+					warning:     true,
+					check: func(context.Context) error {
+						if _, ok := hc.addOns[l5dcharts.PrometheusAddOn]; ok {
+							_, err := hc.kubeAPI.CoreV1().ConfigMaps(hc.ControlPlaneNamespace).Get("linkerd-prometheus-config", metav1.GetOptions{})
+							return err
+						}
+						return &SkipError{Reason: "prometheus add-on not enabled"}
+					},
+				},
+				{
+					description:         "prometheus pod is running",
+					warning:             true,
+					retryDeadline:       hc.RetryDeadline,
+					surfaceErrorOnRetry: true,
+					check: func(context.Context) error {
+						if _, ok := hc.addOns[l5dcharts.PrometheusAddOn]; ok {
+							// populate controlPlanePods to get the latest status, during retries
+							var err error
+							hc.controlPlanePods, err = hc.kubeAPI.GetPodsByNamespace(hc.ControlPlaneNamespace)
+							if err != nil {
+								return err
+							}
+
+							return checkContainerRunning(hc.controlPlanePods, "prometheus")
+						}
+						return &SkipError{Reason: "prometheus add-on not enabled"}
+					},
+				},
+			},
+		},
 		{
 			id: LinkerdGrafanaAddOnChecks,
 			checkers: []checker{

test/integration/install_test.go

@@ -397,7 +397,8 @@ func TestCheckHelmStableBeforeUpgrade(t *testing.T) {
 		t.Skip("Skipping as this is not a helm upgrade test")
 	}
 
-	testCheckCommand(t, "", TestHelper.UpgradeHelmFromVersion(), "", TestHelper.UpgradeHelmFromVersion())
+	// TODO: make checkOutput as true once 2.9 releases
+	testCheckCommand(t, "", TestHelper.UpgradeHelmFromVersion(), "", TestHelper.UpgradeHelmFromVersion(), false)
 }
 
 func TestUpgradeHelm(t *testing.T) {
@@ -584,7 +585,7 @@ func TestVersionPostInstall(t *testing.T) {
 	}
 }
 
-func testCheckCommand(t *testing.T, stage string, expectedVersion string, namespace string, cliVersionOverride string) {
+func testCheckCommand(t *testing.T, stage string, expectedVersion string, namespace string, cliVersionOverride string, compareOutput bool) {
 	var cmd []string
 	var golden string
 	if stage == "proxy" {
@@ -618,6 +619,10 @@ func testCheckCommand(t *testing.T, stage string, expectedVersion string, namesp
 			return fmt.Errorf("'linkerd check' command failed\n%s\n%s", stderr, out)
 		}
 
+		if !compareOutput {
+			return nil
+		}
+
 		err = TestHelper.ValidateOutput(out, golden)
 		if err != nil {
 			return fmt.Errorf("received unexpected output\n%s", err.Error())
@@ -632,11 +637,11 @@ func testCheckCommand(t *testing.T, stage string, expectedVersion string, namesp
 
 // TODO: run this after a `linkerd install config`
 func TestCheckConfigPostInstall(t *testing.T) {
-	testCheckCommand(t, "config", TestHelper.GetVersion(), "", "")
+	testCheckCommand(t, "config", TestHelper.GetVersion(), "", "", true)
 }
 
 func TestCheckPostInstall(t *testing.T) {
-	testCheckCommand(t, "", TestHelper.GetVersion(), "", "")
+	testCheckCommand(t, "", TestHelper.GetVersion(), "", "", true)
 }
 
 func TestUpgradeTestAppWorksAfterUpgrade(t *testing.T) {
@@ -816,7 +821,7 @@ func TestCheckProxy(t *testing.T) {
 		tc := tc // pin
 		t.Run(tc.ns, func(t *testing.T) {
 			prefixedNs := TestHelper.GetTestNamespace(tc.ns)
-			testCheckCommand(t, "proxy", TestHelper.GetVersion(), prefixedNs, "")
+			testCheckCommand(t, "proxy", TestHelper.GetVersion(), prefixedNs, "", true)
 		})
 	}
 }

test/integration/testdata/check.golden

@@ -62,6 +62,12 @@ linkerd-addons
 --------------
 √ 'linkerd-config-addons' config map exists
 
+linkerd-prometheus
+------------------
+√ prometheus add-on service account exists
+√ prometheus add-on config map exists
+√ prometheus pod is running
+
 linkerd-grafana
 ---------------
 √ grafana add-on service account exists

test/integration/testdata/check.multicluster.golden

@@ -62,6 +62,12 @@ linkerd-addons
 --------------
 √ 'linkerd-config-addons' config map exists
 
+linkerd-prometheus
+------------------
+√ prometheus add-on service account exists
+√ prometheus add-on config map exists
+√ prometheus pod is running
+
 linkerd-grafana
 ---------------
 √ grafana add-on service account exists

test/integration/testdata/check.multicluster.proxy.golden

@@ -69,6 +69,12 @@ linkerd-addons
 --------------
 √ 'linkerd-config-addons' config map exists
 
+linkerd-prometheus
+------------------
+√ prometheus add-on service account exists
+√ prometheus add-on config map exists
+√ prometheus pod is running
+
 linkerd-grafana
 ---------------
 √ grafana add-on service account exists

test/integration/testdata/check.proxy.golden

@@ -69,6 +69,12 @@ linkerd-addons
 --------------
 √ 'linkerd-config-addons' config map exists
 
+linkerd-prometheus
+------------------
+√ prometheus add-on service account exists
+√ prometheus add-on config map exists
+√ prometheus pod is running
+
 linkerd-grafana
 ---------------
 √ grafana add-on service account exists