cli: make check return SkipError when there is no prometheus configured (#5150)

Fixes #5143

The availability of prometheus is useful for some calls in public-api
that the check uses. This change updates the ListPods in public-api
to still return the pods even when prometheus is not configured.

For a test that exclusively checks for prometheus metrics, we have a gate
which checks if a prometheus is configured and skips it othervise.

Signed-off-by: Tarun Pothulapati tarunpothulapati@outlook.com

controller/api/public/grpc_server.go

@@ -117,9 +117,10 @@ func (s *grpcServer) ListPods(ctx context.Context, req *pb.ListPodsRequest) (*pb
 
 	// Query Prometheus for all pods present
 	vec, err := s.queryProm(ctx, processStartTimeQuery)
-	if err != nil {
+	if err != nil && !errors.Is(err, ErrNoPrometheusInstance) {
 		return nil, err
 	}
+
 	for _, sample := range vec {
 		pod := string(sample.Metric["pod"])
 		timestamp := sample.Timestamp

controller/api/public/prometheus.go

@@ -42,6 +42,11 @@ const (
 	remoteClusterNameLabel = model.LabelName("target_cluster_name")
 )
 
+var (
+	// ErrNoPrometheusInstance is returned when there is no prometheus instance configured
+	ErrNoPrometheusInstance = errors.New("No prometheus instance to connect")
+)
+
 func extractSampleValue(sample *model.Sample) uint64 {
 	value := uint64(0)
 	if !math.IsNaN(float64(sample.Value)) {
@@ -58,7 +63,7 @@ func (s *grpcServer) queryProm(ctx context.Context, query string) (model.Vector,
 	span.AddAttributes(trace.StringAttribute("queryString", query))
 
 	if s.prometheusAPI == nil {
-		return nil, errors.New("No prometheus instance to connect")
+		return nil, ErrNoPrometheusInstance
 	}
 
 	// single data point (aka summary) query

pkg/healthcheck/healthcheck.go

@@ -1241,6 +1241,16 @@ func (hc *HealthChecker) allCategories() []category {
 							return err
 						}
 
+						// Check if prometheus configured
+						prometheusValues := make(map[string]interface{})
+						err = yaml.Unmarshal(hc.linkerdConfig.Prometheus.Values(), &prometheusValues)
+						if err != nil {
+							return err
+						}
+						if !GetBool(prometheusValues, "enabled") && hc.linkerdConfig.Global.PrometheusURL == "" {
+							return &SkipError{Reason: "no prometheus instance to connect"}
+						}
+
 						return validateDataPlanePodReporting(pods)
 					},
 				},