Triger kube-system HA check based on webhook failure policy (#4861)

This PR changes the HA check that verifies that the `config.linkerd.io/admission-webhooks=disabled` is present on kube-system to be enabled only when the failure policy for the proxy injector webhook is set to `Fail`. This allows users to skip this check in cases when the label is removed because the namespace is managed by the cloud provider like in the case described in #4754

Fix #4754

Signed-off-by: Zahari Dichev <zaharidichev@gmail.com>

pkg/healthcheck/healthcheck.go

@@ -23,6 +23,7 @@ import (
 	"github.com/linkerd/linkerd2/pkg/tls"
 	"github.com/linkerd/linkerd2/pkg/version"
 	log "github.com/sirupsen/logrus"
+	admissionRegistration "k8s.io/api/admissionregistration/v1beta1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -1208,7 +1209,11 @@ func (hc *HealthChecker) allCategories() []category {
 					hintAnchor:  "l5d-injection-disabled",
 					warning:     true,
 					check: func(context.Context) error {
-						if hc.isHA() {
+						policy, err := hc.getMutatingWebhookFailurePolicy()
+						if err != nil {
+							return err
+						}
+						if policy != nil && *policy == admissionRegistration.Fail {
 							return hc.checkHAMetadataPresentOnKubeSystemNamespace()
 						}
 						return &SkipError{Reason: "not run for non HA installs"}
@@ -1667,6 +1672,17 @@ func (hc *HealthChecker) checkCustomResourceDefinitions(shouldExist bool) error
 	return checkResources("CustomResourceDefinitions", objects, []string{"serviceprofiles.linkerd.io"}, shouldExist)
 }
 
+func (hc *HealthChecker) getMutatingWebhookFailurePolicy() (*admissionRegistration.FailurePolicyType, error) {
+	mwc, err := hc.kubeAPI.AdmissionregistrationV1beta1().MutatingWebhookConfigurations().Get(k8s.ProxyInjectorWebhookConfigName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	if len(mwc.Webhooks) != 1 {
+		return nil, fmt.Errorf("expected 1 webhooks, found %d", len(mwc.Webhooks))
+	}
+	return mwc.Webhooks[0].FailurePolicy, nil
+}
+
 func (hc *HealthChecker) checkMutatingWebhookConfigurations(shouldExist bool) error {
 	options := metav1.ListOptions{
 		LabelSelector: hc.controlPlaneComponentsSelector(),
@@ -1885,7 +1901,7 @@ func (hc *HealthChecker) checkHAMetadataPresentOnKubeSystemNamespace() error {
 
 	val, ok := ns.Labels[k8s.AdmissionWebhookLabel]
 	if !ok || val != "disabled" {
-		return fmt.Errorf("kube-system namespace needs to have the label %s: disabled if HA mode is enabled", k8s.AdmissionWebhookLabel)
+		return fmt.Errorf("kube-system namespace needs to have the label %s: disabled if injector webhook failure policy is Fail", k8s.AdmissionWebhookLabel)
 	}
 
 	return nil

pkg/healthcheck/healthcheck_test.go

@@ -2335,17 +2335,8 @@ metadata:
 	})
 }
 
-func getConfigAndKubeSystemNamespace(ha bool, nsLabel string) []string {
+func getWebhookAndKubeSystemNamespace(nsLabel string, failurePolicy string) []string {
 	return []string{fmt.Sprintf(`
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: linkerd-config
-  namespace: linkerd
-data:
-  install: |
-    {"cliVersion":"dev-undefined","flags":[{"name":"ha","value":"%v"}]}`, ha),
-		fmt.Sprintf(`
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -2353,6 +2344,35 @@ metadata:
   labels:
     %s
   name: kube-system`, nsLabel),
+		fmt.Sprintf(`
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: linkerd-proxy-injector-webhook-config
+  labels:
+    linkerd.io/control-plane-component: proxy-injector
+    linkerd.io/control-plane-ns: linkerd
+webhooks:
+- name: linkerd-proxy-injector.linkerd.io
+  namespaceSelector:
+    matchExpressions:
+    - key: config.linkerd.io/admission-webhooks
+      operator: NotIn
+      values:
+      - disabled
+  clientConfig:
+    service:
+      name: linkerd-proxy-injector
+      namespace: linkerd
+      path: "/"
+    caBundle: cHJveHkgaW5qZWN0b3IgQ0EgYnVuZGxl
+  failurePolicy: %s
+  rules:
+  - operations: [ "CREATE" ]
+    apiGroups: [""]
+    apiVersions: ["v1"]
+    resources: ["pods"]
+  sideEffects: None`, failurePolicy),
 	}
 }
 
@@ -2363,24 +2383,24 @@ func TestKubeSystemNamespaceInHA(t *testing.T) {
 		expectedOutput  string
 	}{
 		{
-			"passes when HA is not enabled",
-			getConfigAndKubeSystemNamespace(false, ""),
+			"passes when webhook policy is Ignore is not enabled",
+			getWebhookAndKubeSystemNamespace("", "Ignore"),
 			"",
 		},
 		{
-			"passes when HA is enabled and namespace has required metadata",
-			getConfigAndKubeSystemNamespace(true, "config.linkerd.io/admission-webhooks: disabled"),
+			"passes when webhook policy is Fail and namespace has required metadata",
+			getWebhookAndKubeSystemNamespace("config.linkerd.io/admission-webhooks: disabled", "Fail"),
 			"l5d-injection-disabled pod injection disabled on kube-system",
 		},
 		{
-			"fails when HA and admission hooks are enabled",
-			getConfigAndKubeSystemNamespace(true, "config.linkerd.io/admission-webhooks: enabled"),
-			"l5d-injection-disabled pod injection disabled on kube-system: kube-system namespace needs to have the label config.linkerd.io/admission-webhooks: disabled if HA mode is enabled",
+			"fails when webhook policy is Fail and admission hooks are enabled",
+			getWebhookAndKubeSystemNamespace("config.linkerd.io/admission-webhooks: enabled", "Fail"),
+			"l5d-injection-disabled pod injection disabled on kube-system: kube-system namespace needs to have the label config.linkerd.io/admission-webhooks: disabled if injector webhook failure policy is Fail",
 		},
 		{
-			"fails when HA is enabled and metadata is missing",
-			getConfigAndKubeSystemNamespace(true, ""),
-			"l5d-injection-disabled pod injection disabled on kube-system: kube-system namespace needs to have the label config.linkerd.io/admission-webhooks: disabled if HA mode is enabled",
+			"fails when webhook policy is Fail and metadata is missing",
+			getWebhookAndKubeSystemNamespace("", "Fail"),
+			"l5d-injection-disabled pod injection disabled on kube-system: kube-system namespace needs to have the label config.linkerd.io/admission-webhooks: disabled if injector webhook failure policy is Fail",
 		},
 	}
 
@@ -2391,13 +2411,7 @@ func TestKubeSystemNamespaceInHA(t *testing.T) {
 			hc := NewHealthChecker([]CategoryID{}, &Options{})
 			hc.ControlPlaneNamespace = "linkerd"
 
-			var err error
 			hc.kubeAPI, _ = k8s.NewFakeAPI(tc.k8sConfigs...)
-			_, hc.linkerdConfig, err = hc.checkLinkerdConfigConfigMap()
-			if err != nil {
-				t.Fatalf("Unexpected error: %q", err)
-			}
-
 			hc.addCheckAsCategory("l5d-injection-disabled", LinkerdHAChecks, "pod injection disabled on kube-system")
 
 			obs := newObserver()