viz: check for default-deny in `linkerd viz check --proxy` (#9169)

This branch adds a check to `linkerd viz check --proxy` that checks if
the data plane namespace (or any namespace, if the check is run without
a namespace) has the `config.linkerd.io/default-inbound-policy: deny`
annotation, indicating that the `linkerd-viz` Prometheus instance may
not be authorized to scrape proxies in that namespace.

For example, after installing emojivoto with the default-deny
annotation:

```
linkerd-viz-data-plane
----------------------
√ data plane namespace exists
‼ prometheus is authorized to scrape data plane pods
    prometheus may not be authorized to scrape the following pods:
	* emojivoto/emoji-699d77c79-77w7f
	* emojivoto/voting-55d76f4bcb-6lsml
	* emojivoto/web-6c54d9554d-md2sd
	* emojivoto/vote-bot-b57689ffb-fq8t5
    see https://linkerd.io/2/checks/#l5d-viz-data-plane-prom-authz for hints
```

This check is a warning rather than fatal, because it's possible that
user-created policies may exist that authorize scrapes, which the check
is not currently aware of. We could, potentially, do more exhaustive
checking for whether _any_ policy would authorize scrapes, but that
would require reimplementing a bunch of policy logic inside the `viz`
extension CLI. For now, I settled on making the check a warning, and
having the error message say "prometheus _may_ not be authorized...".
The subsequent check that data plane metrics exist will fail if
Prometheus actually can't scrape anything.

In a subsequent branch, I'll add a `linkerd viz` subcommand for
generating policy resources to allow Prometheus to scrape the proxies in
a namespace; once this is implemented, the check will also check for the
existance of such a policy in that namespace. If the policy does not
exist, the check output will suggest using that command to generate a
policy to allow scrapes.

See #9150

Signed-off-by: Eliza Weisman <eliza@buoyant.io>

pkg/k8s/api.go

@@ -140,7 +140,7 @@ func (kubeAPI *KubernetesAPI) CheckVersion(versionInfo *version.Info) error {
 
 // NamespaceExists validates whether a given namespace exists.
 func (kubeAPI *KubernetesAPI) NamespaceExists(ctx context.Context, namespace string) (bool, error) {
-	ns, err := kubeAPI.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
+	ns, err := kubeAPI.GetNamespace(ctx, namespace)
 	if kerrors.IsNotFound(err) {
 		return false, nil
 	}
@@ -151,6 +151,11 @@ func (kubeAPI *KubernetesAPI) NamespaceExists(ctx context.Context, namespace str
 	return ns != nil, nil
 }
 
+// GetNamespace returns the namespace with a given name, if one exists.
+func (kubeAPI *KubernetesAPI) GetNamespace(ctx context.Context, namespace string) (*corev1.Namespace, error) {
+	return kubeAPI.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
+}
+
 // GetNodes returs all the nodes in a cluster.
 func (kubeAPI *KubernetesAPI) GetNodes(ctx context.Context) ([]corev1.Node, error) {
 	nodes, err := kubeAPI.CoreV1().Nodes().List(ctx, metav1.ListOptions{})

test/integration/upgrade-edge/testdata/check.upgrade.golden

@@ -103,6 +103,7 @@ linkerd-viz
 linkerd-viz-data-plane
 ----------------------
 √ data plane namespace exists
+√ prometheus is authorized to scrape data plane pods
 √ data plane proxy metrics are present in Prometheus
 
 Status check results are √

test/integration/upgrade-stable/testdata/check.upgrade.golden

@@ -103,6 +103,7 @@ linkerd-viz
 linkerd-viz-data-plane
 ----------------------
 √ data plane namespace exists
+√ prometheus is authorized to scrape data plane pods
 √ data plane proxy metrics are present in Prometheus
 
 Status check results are √

viz/pkg/healthcheck/healthcheck.go

@@ -14,6 +14,7 @@ import (
 	"github.com/linkerd/linkerd2/viz/metrics-api/client"
 	pb "github.com/linkerd/linkerd2/viz/metrics-api/gen/viz"
 	"github.com/linkerd/linkerd2/viz/pkg/labels"
+	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apiregistrationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
@@ -301,6 +302,12 @@ func (hc *HealthChecker) VizDataPlaneCategory() *healthcheck.Category {
 				}
 				return hc.CheckNamespace(ctx, hc.DataPlaneNamespace, true)
 			}),
+		*healthcheck.NewChecker("prometheus is authorized to scrape data plane pods").
+			WithHintAnchor("l5d-viz-data-plane-prom-authz").
+			Warning().
+			WithCheck(func(ctx context.Context) error {
+				return hc.checkPromAuthorized(ctx)
+			}),
 		*healthcheck.NewChecker("data plane proxy metrics are present in Prometheus").
 			WithHintAnchor("l5d-data-plane-prom").
 			Warning().
@@ -381,3 +388,74 @@ func fetchTapCaBundle(ctx context.Context, kubeAPI *k8s.KubernetesAPI) ([]*x509.
 	}
 	return caBundle, nil
 }
+
+func (hc *HealthChecker) checkPromAuthorized(ctx context.Context) error {
+	nses, err := hc.getDataPlaneNamespaces(ctx)
+	if err != nil {
+		return err
+	}
+
+	unauthorizedPods := []string{}
+	for _, ns := range nses {
+		// TODO(eliza): check if this namespace has an allow-scrapes config once
+		// that's implemented; if it has one, skip checking its pods.
+		pods, err := hc.KubeAPIClient().GetPodsByNamespace(ctx, ns.GetName())
+		if err != nil {
+			return fmt.Errorf("could not list pods in the %s namespace: %w", ns.GetName(), err)
+		}
+
+		var nsPrefix string
+		if ns.GetName() == hc.DataPlaneNamespace {
+			// if we're only checking one namespace, don't bother appending the
+			// namespace name to the pod's name in the error output
+			nsPrefix = ""
+		} else {
+			// otherwise, include the namespace name as well as the pod's
+			// name, since we are checking all namespaces.
+			nsPrefix = ns.GetName() + "/"
+		}
+
+		for _, pod := range pods {
+			// rather than checking the value of the pod's
+			// `config.linkerd.io/default-inbound-policy` annotation, check the
+			// proxy container's actual env variable. if the cluster-wide
+			// default inbound policy is `deny`, there won't be an override
+			// annotation, but the proxy-injector will have set the env variable
+			// directly.
+			for _, c := range pod.Spec.Containers {
+				if c.Name == k8s.ProxyContainerName {
+					for _, env := range c.Env {
+						if env.Name == "LINKERD2_PROXY_INBOUND_DEFAULT_POLICY" && env.Value == "deny" {
+							unauthorizedPods = append(unauthorizedPods, fmt.Sprintf("\t* %s%s", nsPrefix, pod.Name))
+							break
+						}
+					}
+					break
+				}
+			}
+		}
+	}
+
+	if len(unauthorizedPods) > 0 {
+		podList := strings.Join(unauthorizedPods, "\n")
+		return fmt.Errorf("prometheus may not be authorized to scrape the following pods:\n%s", podList)
+	}
+
+	return nil
+}
+
+func (hc *HealthChecker) getDataPlaneNamespaces(ctx context.Context) ([]corev1.Namespace, error) {
+	if hc.DataPlaneNamespace != "" {
+		ns, err := hc.KubeAPIClient().GetNamespace(ctx, hc.DataPlaneNamespace)
+		if err != nil {
+			return nil, err
+		}
+		return []corev1.Namespace{*ns}, nil
+	}
+
+	nses, err := hc.KubeAPIClient().CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return nses.Items, nil
+}