feat(multicluster): have linkerd-multicluster chart be responsible for service mirror controllers - CLI (#13782)

Followup to #13770 and #13781, based off of branch `alpeb/multicluster-chart-manage-smc-probes`

Addresses CLI tasks listed in #13768

- Refers in `multicluster/cmd/install.go` the new templates added in #13770, allowing `linkerd mc install` to also install the model that was implemented in #13770 just for helm.
- Adds a new flag `linkerd mc link --service-mirror`; if `false` then it only outputs the Link CR and the credentials secret. The default is `true` so the new model is opt-in.
- Refactors `linkerd mc check` in order to check for resources from both the old and new models.
- Extend the image registry overriding logic to cover the new controllers as well.

multicluster/cmd/check.go

@@ -36,11 +36,22 @@ const (
 
 	// LinkerdMulticlusterExtensionCheck adds checks related to the multicluster extension
 	LinkerdMulticlusterExtensionCheck healthcheck.CategoryID = "linkerd-multicluster"
+)
 
-	linkerdServiceMirrorServiceAccountName = "linkerd-service-mirror-%s"
+// For these vars, the second name is for service mirror controllers
-	linkerdServiceMirrorComponentName      = "service-mirror"
+// managed by the linkerd-multicluster chart
-	linkerdServiceMirrorClusterRoleName    = "linkerd-service-mirror-access-local-resources-%s"
+var (
-	linkerdServiceMirrorRoleName           = "linkerd-service-mirror-read-remote-creds-%s"
+	linkerdServiceMirrorServiceAccountNames = []string{"linkerd-service-mirror-%s", "controller-%s"}
+	linkerdServiceMirrorComponentNames      = []string{"service-mirror", "controller"}
+
+	linkerdServiceMirrorClusterRoleNames = []string{
+		"linkerd-service-mirror-access-local-resources-%s",
+		"linkerd-multicluster-controller-access-local-resources",
+	}
+	linkerdServiceMirrorRoleNames = []string{
+		"linkerd-service-mirror-read-remote-creds-%s",
+		"controller-read-remote-creds-%s",
+	}
 )
 
 type checkOptions struct {
@@ -480,54 +491,101 @@ func (hc *healthChecker) checkServiceMirrorLocalRBAC(ctx context.Context) error
 		err := healthcheck.CheckServiceAccounts(
 			ctx,
 			hc.KubeAPIClient(),
-			[]string{fmt.Sprintf(linkerdServiceMirrorServiceAccountName, link.Spec.TargetClusterName)},
+			[]string{fmt.Sprintf(linkerdServiceMirrorServiceAccountNames[0], link.Spec.TargetClusterName)},
 			link.Namespace,
 			serviceMirrorComponentsSelector(link.Spec.TargetClusterName),
 		)
 		if err != nil {
-			messages = append(messages, err.Error())
+			err2 := healthcheck.CheckServiceAccounts(
+				ctx,
+				hc.KubeAPIClient(),
+				[]string{fmt.Sprintf(linkerdServiceMirrorServiceAccountNames[1], link.Spec.TargetClusterName)},
+				link.Namespace,
+				serviceMirrorComponentsSelector(link.Spec.TargetClusterName),
+			)
+			if err2 != nil {
+				messages = append(messages, err.Error(), err2.Error())
+			}
 		}
 		err = healthcheck.CheckClusterRoles(
 			ctx,
 			hc.KubeAPIClient(),
 			true,
-			[]string{fmt.Sprintf(linkerdServiceMirrorClusterRoleName, link.Spec.TargetClusterName)},
+			[]string{fmt.Sprintf(linkerdServiceMirrorClusterRoleNames[0], link.Spec.TargetClusterName)},
 			serviceMirrorComponentsSelector(link.Spec.TargetClusterName),
 		)
 		if err != nil {
-			messages = append(messages, err.Error())
+			err2 := healthcheck.CheckClusterRoles(
+				ctx,
+				hc.KubeAPIClient(),
+				true,
+				[]string{linkerdServiceMirrorClusterRoleNames[1]},
+				"component=controller",
+			)
+			if err2 != nil {
+				messages = append(messages, err.Error(), err2.Error())
+			}
 		}
 		err = healthcheck.CheckClusterRoleBindings(
 			ctx,
 			hc.KubeAPIClient(),
 			true,
-			[]string{fmt.Sprintf(linkerdServiceMirrorClusterRoleName, link.Spec.TargetClusterName)},
+			[]string{fmt.Sprintf(linkerdServiceMirrorClusterRoleNames[0], link.Spec.TargetClusterName)},
 			serviceMirrorComponentsSelector(link.Spec.TargetClusterName),
 		)
 		if err != nil {
-			messages = append(messages, err.Error())
+			err2 := healthcheck.CheckClusterRoleBindings(
+				ctx,
+				hc.KubeAPIClient(),
+				true,
+				[]string{fmt.Sprintf("%s-%s", linkerdServiceMirrorClusterRoleNames[1], link.Spec.TargetClusterName)},
+				serviceMirrorComponentsSelector(link.Spec.TargetClusterName),
+			)
+			if err2 != nil {
+				messages = append(messages, err.Error(), err2.Error())
+			}
 		}
 		err = healthcheck.CheckRoles(
 			ctx,
 			hc.KubeAPIClient(),
 			true,
 			link.Namespace,
-			[]string{fmt.Sprintf(linkerdServiceMirrorRoleName, link.Spec.TargetClusterName)},
+			[]string{fmt.Sprintf(linkerdServiceMirrorRoleNames[0], link.Spec.TargetClusterName)},
 			serviceMirrorComponentsSelector(link.Spec.TargetClusterName),
 		)
 		if err != nil {
-			messages = append(messages, err.Error())
+			err2 := healthcheck.CheckRoles(
+				ctx,
+				hc.KubeAPIClient(),
+				true,
+				link.Namespace,
+				[]string{fmt.Sprintf(linkerdServiceMirrorRoleNames[1], link.Spec.TargetClusterName)},
+				serviceMirrorComponentsSelector(link.Spec.TargetClusterName),
+			)
+			if err2 != nil {
+				messages = append(messages, err.Error(), err2.Error())
+			}
 		}
 		err = healthcheck.CheckRoleBindings(
 			ctx,
 			hc.KubeAPIClient(),
 			true,
 			link.Namespace,
-			[]string{fmt.Sprintf(linkerdServiceMirrorRoleName, link.Spec.TargetClusterName)},
+			[]string{fmt.Sprintf(linkerdServiceMirrorRoleNames[0], link.Spec.TargetClusterName)},
 			serviceMirrorComponentsSelector(link.Spec.TargetClusterName),
 		)
 		if err != nil {
-			messages = append(messages, err.Error())
+			err2 := healthcheck.CheckRoleBindings(
+				ctx,
+				hc.KubeAPIClient(),
+				true,
+				link.Namespace,
+				[]string{fmt.Sprintf(linkerdServiceMirrorRoleNames[1], link.Spec.TargetClusterName)},
+				serviceMirrorComponentsSelector(link.Spec.TargetClusterName),
+			)
+			if err2 != nil {
+				messages = append(messages, err.Error(), err2.Error())
+			}
 		}
 		links = append(links, fmt.Sprintf("\t* %s", link.Spec.TargetClusterName))
 	}
@@ -611,7 +669,7 @@ func (hc *healthChecker) checkIfGatewayMirrorsHaveEndpoints(ctx context.Context,
 
 		// Get the service mirror component in the linkerd-multicluster
 		// namespace which corresponds to the current link.
-		selector = metav1.ListOptions{LabelSelector: fmt.Sprintf("component=linkerd-service-mirror,mirror.linkerd.io/cluster-name=%s", link.Spec.TargetClusterName)}
+		selector = metav1.ListOptions{LabelSelector: fmt.Sprintf("component in(linkerd-service-mirror, controller),mirror.linkerd.io/cluster-name=%s", link.Spec.TargetClusterName)}
 		pods, err := hc.KubeAPIClient().CoreV1().Pods(multiclusterNs.Name).List(ctx, selector)
 		if err != nil {
 			errors = append(errors, fmt.Errorf("failed to get the service-mirror component for target cluster %s: %w", link.Spec.TargetClusterName, err))
@@ -742,7 +800,7 @@ func joinErrors(errs []error, tabDepth int) error {
 }
 
 func serviceMirrorComponentsSelector(targetCluster string) string {
-	return fmt.Sprintf("component=%s,%s=%s",
+	return fmt.Sprintf("component in (%s),%s=%s",
-		linkerdServiceMirrorComponentName,
+		strings.Join(linkerdServiceMirrorComponentNames, ", "),
 		k8s.RemoteClusterNameLabel, targetCluster)
 }

multicluster/cmd/install.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"strings"
 	"time"
 
 	"github.com/linkerd/linkerd2/multicluster/static"
@@ -46,6 +47,11 @@ var TemplatesMulticluster = []string{
 	"templates/link-crd.yaml",
 	"templates/service-mirror-policy.yaml",
 	"templates/local-service-mirror.yaml",
+	"templates/controller-clusterrole.yaml",
+	"templates/controller/deployment.yaml",
+	"templates/controller/pdb.yaml",
+	"templates/controller/probe-svc.yaml",
+	"templates/controller/rbac.yaml",
 }
 
 func newMulticlusterInstallCommand() *cobra.Command {
@@ -184,6 +190,18 @@ func render(w io.Writer, values *multicluster.Values, valuesOverrides map[string
 		return err
 	}
 
+	if reg := os.Getenv(flags.EnvOverrideDockerRegistry); reg != "" {
+		overrideRegistry(chart.Values, "localServiceMirror.image.name", reg, "cr.l5d.io/linkerd/controller")
+
+		if controllers, ok := valuesOverrides["controllers"].([]any); ok {
+			for _, c := range controllers {
+				if controller, ok := c.(map[string]any); ok {
+					overrideRegistry(controller, "image.name", reg, "cr.l5d.io/linkerd/controller")
+				}
+			}
+		}
+	}
+
 	vals, err := chartutil.CoalesceValues(chart, valuesOverrides)
 	if err != nil {
 		return err
@@ -233,11 +251,8 @@ func buildMulticlusterInstallValues(ctx context.Context, opts *multiclusterInsta
 		return nil, err
 	}
 
-	if reg := os.Getenv(flags.EnvOverrideDockerRegistry); reg != "" {
-		defaults.LocalServiceMirror.Image.Name = pkgcmd.RegistryOverride(defaults.LocalServiceMirror.Image.Name, reg)
-	}
-
 	defaults.LocalServiceMirror.Image.Version = version.Version
+	defaults.ControllerDefaults.Image.Version = version.Version
 	defaults.Gateway.Enabled = opts.gateway.Enabled
 	defaults.Gateway.Port = opts.gateway.Port
 	defaults.Gateway.Probe.Seconds = opts.gateway.Probe.Seconds
@@ -263,3 +278,31 @@ func buildMulticlusterInstallValues(ctx context.Context, opts *multiclusterInsta
 
 	return defaults, nil
 }
+
+// overrideRegistry overrides the image name in the values map at the given
+// keyPath with the given registry. If the keyPath does not exist, it is
+// created.
+func overrideRegistry(values map[string]any, keyPath, reg, defaultImage string) {
+	keys := strings.Split(keyPath, ".")
+	m := values
+
+	// Traverse map using keys, except for the last key
+	for _, key := range keys[:len(keys)-1] {
+		if next, ok := m[key].(map[string]any); ok {
+			m = next
+		} else {
+			newMap := make(map[string]any)
+			m[key] = newMap
+			m = newMap
+		}
+	}
+
+	// Override the last key if it exists and is a string
+	lastKey := keys[len(keys)-1]
+	if val, ok := m[lastKey].(string); ok {
+		m[lastKey] = pkgcmd.RegistryOverride(val, reg)
+	} else {
+		// If the key is missing, initialize it with an empty string before overriding
+		m[lastKey] = pkgcmd.RegistryOverride(defaultImage, reg)
+	}
+}

multicluster/cmd/link.go

@@ -60,6 +60,7 @@ type (
 		gatewayPort              uint32
 		ha                       bool
 		enableGateway            bool
+		enableServiceMirror      bool
 		output                   string
 	}
 )
@@ -372,8 +373,10 @@ A full list of configurable values can be found at https://github.com/linkerd/li
 			stdout.Write(separator)
 			stdout.Write(linkOut)
 			stdout.Write(separator)
-			stdout.Write(serviceMirrorOut)
+			if opts.enableServiceMirror {
-			stdout.Write(separator)
+				stdout.Write(serviceMirrorOut)
+				stdout.Write(separator)
+			}
 
 			return nil
 		},
@@ -399,6 +402,7 @@ A full list of configurable values can be found at https://github.com/linkerd/li
 	cmd.Flags().Uint32Var(&opts.gatewayPort, "gateway-port", opts.gatewayPort, "If specified, overwrites gateway port when gateway service is not type LoadBalancer")
 	cmd.Flags().BoolVar(&opts.ha, "ha", opts.ha, "Enable HA configuration for the service-mirror deployment (default false)")
 	cmd.Flags().BoolVar(&opts.enableGateway, "gateway", opts.enableGateway, "If false, allows a link to be created against a cluster that does not have a gateway service")
+	cmd.Flags().BoolVar(&opts.enableServiceMirror, "service-mirror", opts.enableServiceMirror, "If false, only outputs link manifest and credentials secrets")
 	cmd.Flags().StringVarP(&opts.output, "output", "o", "yaml", "Output format. One of: json|yaml")
 
 	pkgcmd.ConfigureNamespaceFlagCompletion(
@@ -506,7 +510,7 @@ func newLinkOptionsWithDefault() (*linkOptions, error) {
 		gatewayPort:              0,
 		ha:                       false,
 		enableGateway:            true,
-		output:                   "yaml",
+		enableServiceMirror:      true,
 	}, nil
 }
 

multicluster/cmd/testdata/install_default.golden

@@ -961,3 +961,21 @@ spec:
                   apiVersion: v1
                   fieldPath: metadata.namespace
                 path: namespace
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: linkerd-multicluster-controller-access-local-resources
+  labels:
+    linkerd.io/extension: multicluster
+    component: controller
+rules:
+- apiGroups: [""]
+  resources: ["endpoints", "services"]
+  verbs: ["list", "get", "watch", "create", "delete", "update"]
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["list", "get", "watch"]
+
+
+

multicluster/cmd/testdata/install_ha.golden

@@ -1071,3 +1071,21 @@ spec:
   selector:
     matchLabels:
       component: local-service-mirror
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: linkerd-multicluster-controller-access-local-resources
+  labels:
+    linkerd.io/extension: multicluster
+    component: controller
+rules:
+- apiGroups: [""]
+  resources: ["endpoints", "services"]
+  verbs: ["list", "get", "watch", "create", "delete", "update"]
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["list", "get", "watch"]
+
+
+

multicluster/cmd/testdata/install_psp.golden

@@ -995,3 +995,21 @@ spec:
                   apiVersion: v1
                   fieldPath: metadata.namespace
                 path: namespace
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: linkerd-multicluster-controller-access-local-resources
+  labels:
+    linkerd.io/extension: multicluster
+    component: controller
+rules:
+- apiGroups: [""]
+  resources: ["endpoints", "services"]
+  verbs: ["list", "get", "watch", "create", "delete", "update"]
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["list", "get", "watch"]
+
+
+

multicluster/values/values.go

@@ -45,6 +45,7 @@ type Values struct {
 	ServiceMirrorExperimentalEnv []corev1.EnvVar `json:"serviceMirrorExperimentalEnv"`
 
 	LocalServiceMirror *LocalServiceMirror `json:"localServiceMirror"`
+	ControllerDefaults *ControllerDefaults `json:"controllerDefaults"`
 }
 
 // Gateway contains all options related to the Gateway Service
@@ -85,6 +86,31 @@ type LocalServiceMirror struct {
 	GID                      int64           `json:"GID"`
 }
 
+// ControllerDefaults contains all the entries for the controllerDefaults
+// section that are not empty by default
+type ControllerDefaults struct {
+	Replicas               uint32                     `json:"replicas"`
+	Image                  *linkerd2.Image            `json:"image"`
+	Gateway                *ControllerDefaultsGateway `json:"gateway"`
+	LogLevel               string                     `json:"logLevel"`
+	LogFormat              string                     `json:"logFormat"`
+	EnableHeadlessServices bool                       `json:"enableHeadlessServices"`
+	EnablePprof            bool                       `json:"enablePprof"`
+	UID                    int64                      `json:"UID"`
+	GID                    int64                      `json:"GID"`
+	RetryLimit             uint32                     `json:"retryLimit"`
+	EnablePodAntiAffinity  bool                       `json:"enablePodAntiAffinity"`
+}
+
+type ControllerDefaultsGateway struct {
+	Enabled bool                     `json:"enabled"`
+	Probe   *ControllerDefaultsProbe `json:"probe"`
+}
+
+type ControllerDefaultsProbe struct {
+	Port uint32 `json:"port"`
+}
+
 // NewInstallValues returns a new instance of the Values type.
 func NewInstallValues() (*Values, error) {
 	chartDir := fmt.Sprintf("%s/", helmDefaultChartDir)