Warn when webhook certificates near expiry (#5155)

Fixes #5149 

Before:

```
linkerd-webhooks-and-apisvc-tls
-------------------------------
× tap API server has valid cert
    certificate will expire on 2020-10-28T20:22:32Z
    see https://linkerd.io/checks/#l5d-tap-cert-valid for hints
```

After:

```
linkerd-webhooks-and-apisvc-tls
-------------------------------
√ tap API server has valid cert
‼ tap API server cert is valid for at least 60 days
    certificate will expire on 2020-10-28T20:22:32Z
    see https://linkerd.io/checks/#l5d-webhook-cert-not-expiring-soon for hints
√ proxy-injector webhook has valid cert
‼ proxy-injector cert is valid for at least 60 days
    certificate will expire on 2020-10-29T18:17:03Z
    see https://linkerd.io/checks/#l5d-webhook-cert-not-expiring-soon for hints
√ sp-validator webhook has valid cert
‼ sp-validator cert is valid for at least 60 days
    certificate will expire on 2020-10-28T20:21:34Z
    see https://linkerd.io/checks/#l5d-webhook-cert-not-expiring-soon for hints
```

Signed-off-by: Alex Leong <alex@buoyant.io>

pkg/healthcheck/healthcheck.go

@@ -1051,6 +1051,22 @@ func (hc *HealthChecker) allCategories() []category {
 						return hc.checkCertAndAnchors(cert, anchors, identityName)
 					},
 				},
+				{
+					description: "tap API server cert is valid for at least 60 days",
+					warning:     true,
+					hintAnchor:  "l5d-webhook-cert-not-expiring-soon",
+					check: func(ctx context.Context) error {
+						cert, err := hc.fetchCredsFromSecret(ctx, tapTLSSecretName)
+						if kerrors.IsNotFound(err) {
+							cert, err = hc.fetchCredsFromOldSecret(ctx, tapOldTLSSecretName)
+						}
+						if err != nil {
+							return err
+						}
+						return hc.checkCertAndAnchorsExpiringSoon(cert)
+
+					},
+				},
 				{
 					description: "proxy-injector webhook has valid cert",
 					hintAnchor:  "l5d-proxy-injector-webhook-cert-valid",
@@ -1072,6 +1088,22 @@ func (hc *HealthChecker) allCategories() []category {
 						return hc.checkCertAndAnchors(cert, anchors, identityName)
 					},
 				},
+				{
+					description: "proxy-injector cert is valid for at least 60 days",
+					warning:     true,
+					hintAnchor:  "l5d-webhook-cert-not-expiring-soon",
+					check: func(ctx context.Context) error {
+						cert, err := hc.fetchCredsFromSecret(ctx, proxyInjectorTLSSecretName)
+						if kerrors.IsNotFound(err) {
+							cert, err = hc.fetchCredsFromOldSecret(ctx, proxyInjectorOldTLSSecretName)
+						}
+						if err != nil {
+							return err
+						}
+						return hc.checkCertAndAnchorsExpiringSoon(cert)
+
+					},
+				},
 				{
 					description: "sp-validator webhook has valid cert",
 					hintAnchor:  "l5d-sp-validator-webhook-cert-valid",
@@ -1092,6 +1124,22 @@ func (hc *HealthChecker) allCategories() []category {
 						return hc.checkCertAndAnchors(cert, anchors, identityName)
 					},
 				},
+				{
+					description: "sp-validator cert is valid for at least 60 days",
+					warning:     true,
+					hintAnchor:  "l5d-webhook-cert-not-expiring-soon",
+					check: func(ctx context.Context) error {
+						cert, err := hc.fetchCredsFromSecret(ctx, spValidatorTLSSecretName)
+						if kerrors.IsNotFound(err) {
+							cert, err = hc.fetchCredsFromOldSecret(ctx, spValidatorOldTLSSecretName)
+						}
+						if err != nil {
+							return err
+						}
+						return hc.checkCertAndAnchorsExpiringSoon(cert)
+
+					},
+				},
 			},
 		},
 		{
@@ -1346,9 +1394,22 @@ func (hc *HealthChecker) checkCertAndAnchors(cert *tls.Cred, trustAnchors []*x50
 		return fmt.Errorf("Anchors not within their validity period:\n\t%s", strings.Join(expiredAnchors, "\n\t"))
 	}
 
+	// check cert validity
+	if err := issuercerts.CheckCertValidityPeriod(cert.Certificate); err != nil {
+		return fmt.Errorf("certificate is %s", err)
+	}
+
+	if err := cert.Verify(tls.CertificatesToPool(trustAnchors), identityName, time.Time{}); err != nil {
+		return fmt.Errorf("cert is not issued by the trust anchor: %s", err)
+	}
+
+	return nil
+}
+
+func (hc *HealthChecker) checkCertAndAnchorsExpiringSoon(cert *tls.Cred) error {
 	// check anchors not expiring soon
 	var expiringAnchors []string
-	for _, anchor := range trustAnchors {
+	for _, anchor := range cert.TrustChain {
 		anchor := anchor
 		if err := issuercerts.CheckExpiringSoon(anchor); err != nil {
 			expiringAnchors = append(expiringAnchors, fmt.Sprintf("* %v %s %s", anchor.SerialNumber, anchor.Subject.CommonName, err))
@@ -1358,20 +1419,10 @@ func (hc *HealthChecker) checkCertAndAnchors(cert *tls.Cred, trustAnchors []*x50
 		return fmt.Errorf("Anchors expiring soon:\n\t%s", strings.Join(expiringAnchors, "\n\t"))
 	}
 
-	// check cert validity
-	if err := issuercerts.CheckCertValidityPeriod(cert.Certificate); err != nil {
-		return fmt.Errorf("certificate is %s", err)
-	}
-
 	// check cert not expiring soon
 	if err := issuercerts.CheckExpiringSoon(cert.Certificate); err != nil {
 		return fmt.Errorf("certificate %s", err)
 	}
-
-	if err := cert.Verify(tls.CertificatesToPool(trustAnchors), identityName, time.Time{}); err != nil {
-		return fmt.Errorf("cert is not issued by the trust anchor: %s", err)
-	}
-
 	return nil
 }
 

test/integration/testdata/check.cni.golden

@@ -55,8 +55,11 @@ linkerd-identity
 linkerd-webhooks-and-apisvc-tls
 -------------------------------
 √ tap API server has valid cert
+√ tap API server cert is valid for at least 60 days
 √ proxy-injector webhook has valid cert
+√ proxy-injector cert is valid for at least 60 days
 √ sp-validator webhook has valid cert
+√ sp-validator cert is valid for at least 60 days
 
 linkerd-api
 -----------

test/integration/testdata/check.cni.proxy.golden

@@ -55,8 +55,11 @@ linkerd-identity
 linkerd-webhooks-and-apisvc-tls
 -------------------------------
 √ tap API server has valid cert
+√ tap API server cert is valid for at least 60 days
 √ proxy-injector webhook has valid cert
+√ proxy-injector cert is valid for at least 60 days
 √ sp-validator webhook has valid cert
+√ sp-validator cert is valid for at least 60 days
 
 linkerd-identity-data-plane
 ---------------------------

test/integration/testdata/check.golden

@@ -43,8 +43,11 @@ linkerd-identity
 linkerd-webhooks-and-apisvc-tls
 -------------------------------
 √ tap API server has valid cert
+√ tap API server cert is valid for at least 60 days
 √ proxy-injector webhook has valid cert
+√ proxy-injector cert is valid for at least 60 days
 √ sp-validator webhook has valid cert
+√ sp-validator cert is valid for at least 60 days
 
 linkerd-api
 -----------

test/integration/testdata/check.multicluster.golden

@@ -43,8 +43,11 @@ linkerd-identity
 linkerd-webhooks-and-apisvc-tls
 -------------------------------
 √ tap API server has valid cert
+√ tap API server cert is valid for at least 60 days
 √ proxy-injector webhook has valid cert
+√ proxy-injector cert is valid for at least 60 days
 √ sp-validator webhook has valid cert
+√ sp-validator cert is valid for at least 60 days
 
 linkerd-api
 -----------

test/integration/testdata/check.multicluster.proxy.golden

@@ -43,8 +43,11 @@ linkerd-identity
 linkerd-webhooks-and-apisvc-tls
 -------------------------------
 √ tap API server has valid cert
+√ tap API server cert is valid for at least 60 days
 √ proxy-injector webhook has valid cert
+√ proxy-injector cert is valid for at least 60 days
 √ sp-validator webhook has valid cert
+√ sp-validator cert is valid for at least 60 days
 
 linkerd-identity-data-plane
 ---------------------------

test/integration/testdata/check.proxy.golden

@@ -43,8 +43,11 @@ linkerd-identity
 linkerd-webhooks-and-apisvc-tls
 -------------------------------
 √ tap API server has valid cert
+√ tap API server cert is valid for at least 60 days
 √ proxy-injector webhook has valid cert
+√ proxy-injector cert is valid for at least 60 days
 √ sp-validator webhook has valid cert
+√ sp-validator cert is valid for at least 60 days
 
 linkerd-identity-data-plane
 ---------------------------