cli: Update 'check' command to validate HA configuration (#3942)

Add check for number of control plane replicas for HA

Signed-off-by: Mayank Shah <mayankshah1614@gmail.com>

pkg/healthcheck/healthcheck.go

@@ -166,6 +166,15 @@ const HintBaseURL = "https://linkerd.io/checks/#"
 // TODO: Make this default value overridiable, e.g. by CLI flag
 const AllowedClockSkew = time.Minute + tls.DefaultClockSkewAllowance
 
+var linkerdHAControlPlaneComponents = []string{
+	"linkerd-controller",
+	"linkerd-destination",
+	"linkerd-identity",
+	"linkerd-proxy-injector",
+	"linkerd-sp-validator",
+	"linkerd-tap",
+}
+
 var (
 	retryWindow    = 5 * time.Second
 	requestTimeout = 30 * time.Second
@@ -1167,11 +1176,43 @@ func (hc *HealthChecker) allCategories() []category {
 						return &SkipError{Reason: "not run for non HA installs"}
 					},
 				},
+				{
+					description:   "multiple replicas of control plane pods",
+					hintAnchor:    "l5d-control-plane-replicas",
+					retryDeadline: hc.RetryDeadline,
+					warning:       true,
+					check: func(ctx context.Context) error {
+						if hc.isHA() {
+							return hc.checkMinReplicasAvailable()
+						}
+						return &SkipError{Reason: "not run for non HA installs"}
+					},
+				},
 			},
 		},
 	}
 }
 
+func (hc *HealthChecker) checkMinReplicasAvailable() error {
+	faulty := []string{}
+
+	for _, component := range linkerdHAControlPlaneComponents {
+		conf, err := hc.kubeAPI.AppsV1().Deployments(hc.ControlPlaneNamespace).Get(component, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		if conf.Status.AvailableReplicas <= 1 {
+			faulty = append(faulty, component)
+		}
+	}
+
+	if len(faulty) > 0 {
+		return fmt.Errorf("not enough replicas available for %v", faulty)
+	}
+	return nil
+}
+
 func (hc *HealthChecker) issuerIdentity() string {
 	return fmt.Sprintf("identity.%s.%s", hc.ControlPlaneNamespace, hc.linkerdConfig.Global.IdentityContext.TrustDomain)
 }

pkg/healthcheck/healthcheck_test.go

@@ -3029,3 +3029,125 @@ func TestCniChecks(t *testing.T) {
 	}
 
 }
+
+func TestMinReplicaCheck(t *testing.T) {
+	hc := NewHealthChecker(
+		[]CategoryID{LinkerdHAChecks},
+		&Options{
+			ControlPlaneNamespace: "linkerd",
+		},
+	)
+
+	var err error
+
+	testCases := []struct {
+		controlPlaneResourceDefs []string
+		expected                 error
+	}{
+		{
+			controlPlaneResourceDefs: generateAllControlPlaneDef(&controlPlaneReplicaOptions{
+				controller:    1,
+				destination:   3,
+				identity:      3,
+				proxyInjector: 3,
+				spValidator:   1,
+				tap:           3,
+			}, t),
+			expected: fmt.Errorf("not enough replicas available for [linkerd-controller linkerd-sp-validator]"),
+		},
+		{
+			controlPlaneResourceDefs: generateAllControlPlaneDef(&controlPlaneReplicaOptions{
+				controller:    3,
+				destination:   2,
+				identity:      1,
+				proxyInjector: 1,
+				spValidator:   0,
+				tap:           3,
+			}, t),
+			expected: fmt.Errorf("not enough replicas available for [linkerd-identity linkerd-proxy-injector linkerd-sp-validator]"),
+		},
+		{
+			controlPlaneResourceDefs: generateAllControlPlaneDef(&controlPlaneReplicaOptions{
+				controller:    3,
+				destination:   2,
+				identity:      2,
+				proxyInjector: 3,
+				spValidator:   2,
+				tap:           3,
+			}, t),
+			expected: nil,
+		},
+	}
+
+	for i, tc := range testCases {
+		tc := tc //pin
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			hc.kubeAPI, err = k8s.NewFakeAPI(tc.controlPlaneResourceDefs...)
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = hc.checkMinReplicasAvailable()
+			if err == nil && tc.expected != nil {
+				t.Log("Expected error: nil")
+				t.Logf("Received error: %s\n", err)
+				t.Fatal("test case failed")
+			}
+			if err != nil {
+				if err.Error() != tc.expected.Error() {
+					t.Logf("Expected error: %s\n", tc.expected)
+					t.Logf("Received error: %s\n", err)
+					t.Fatal("test case failed")
+				}
+			}
+		})
+	}
+}
+
+type controlPlaneReplicaOptions struct {
+	controller    int
+	destination   int
+	identity      int
+	proxyInjector int
+	spValidator   int
+	tap           int
+}
+
+func getSingleControlPlaneDef(component string, availableReplicas int) string {
+	return fmt.Sprintf(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: %s
+  namespace: linkerd
+spec:
+  template:
+    spec:
+      containers:
+        - image: "hello-world"
+          name: test
+status:
+  availableReplicas: %d`, component, availableReplicas)
+}
+
+func generateAllControlPlaneDef(replicaOptions *controlPlaneReplicaOptions, t *testing.T) []string {
+	resourceDefs := []string{}
+	for _, component := range linkerdHAControlPlaneComponents {
+		switch component {
+		case "linkerd-controller":
+			resourceDefs = append(resourceDefs, getSingleControlPlaneDef(component, replicaOptions.controller))
+		case "linkerd-destination":
+			resourceDefs = append(resourceDefs, getSingleControlPlaneDef(component, replicaOptions.destination))
+		case "linkerd-identity":
+			resourceDefs = append(resourceDefs, getSingleControlPlaneDef(component, replicaOptions.identity))
+		case "linkerd-sp-validator":
+			resourceDefs = append(resourceDefs, getSingleControlPlaneDef(component, replicaOptions.spValidator))
+		case "linkerd-proxy-injector":
+			resourceDefs = append(resourceDefs, getSingleControlPlaneDef(component, replicaOptions.proxyInjector))
+		case "linkerd-tap":
+			resourceDefs = append(resourceDefs, getSingleControlPlaneDef(component, replicaOptions.tap))
+		default:
+			t.Fatal("Could not find the resource")
+		}
+	}
+	return resourceDefs
+}