package cmd import ( "bytes" "encoding/json" "errors" "fmt" "os" "path" "strconv" "strings" "time" "github.com/linkerd/linkerd2/controller/gen/apis/link/v1alpha3" "github.com/linkerd/linkerd2/multicluster/static" multicluster "github.com/linkerd/linkerd2/multicluster/values" "github.com/linkerd/linkerd2/pkg/charts" partials "github.com/linkerd/linkerd2/pkg/charts/static" pkgcmd "github.com/linkerd/linkerd2/pkg/cmd" "github.com/linkerd/linkerd2/pkg/flags" "github.com/linkerd/linkerd2/pkg/k8s" "github.com/linkerd/linkerd2/pkg/version" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" chartloader "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" valuespkg "helm.sh/helm/v3/pkg/cli/values" "helm.sh/helm/v3/pkg/engine" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/yaml" ) const ( clusterNameLabel = "multicluster.linkerd.io/cluster-name" trustDomainAnnotation = "multicluster.linkerd.io/trust-domain" clusterDomainAnnotation = "multicluster.linkerd.io/cluster-domain" ) type ( linkOptions struct { namespace string clusterName string apiServerAddress string serviceAccountName string gatewayName string gatewayNamespace string serviceMirrorRetryLimit uint32 logLevel string logFormat string controlPlaneVersion string dockerRegistry string selector string remoteDiscoverySelector string federatedServiceSelector string gatewayAddresses string gatewayPort uint32 ha bool enableGateway bool enableServiceMirror bool output string } ) func newLinkCommand() *cobra.Command { opts, err := newLinkOptionsWithDefault() // Override the default value with env registry path. // If cli cmd contains --registry flag, it will override env variable. if registry := os.Getenv(flags.EnvOverrideDockerRegistry); registry != "" { opts.dockerRegistry = registry } var valuesOptions valuespkg.Options if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } cmd := &cobra.Command{ Use: "link", Short: "Outputs resources that allow another cluster to mirror services from this one", Long: `Outputs resources that allow another cluster to mirror services from this one. Note that the Link resource applies only in one direction. In order for two clusters to mirror each other, a Link resource will have to be generated for each cluster and applied to the other.`, Args: cobra.NoArgs, Example: ` # To link the west cluster to east linkerd --context=east multicluster link --cluster-name east | kubectl --context=west apply -f - The command can be configured by using the --set, --values, --set-string and --set-file flags. A full list of configurable values can be found at https://github.com/linkerd/linkerd2/blob/main/multicluster/charts/linkerd-multicluster-link/README.md `, RunE: func(cmd *cobra.Command, args []string) error { if opts.clusterName == "" { return errors.New("You need to specify cluster name") } configMap, err := getLinkerdConfigMap(cmd.Context()) if err != nil { if kerrors.IsNotFound(err) { return errors.New("you need Linkerd to be installed on a cluster in order to get its credentials") } return err } rules := clientcmd.NewDefaultClientConfigLoadingRules() rules.ExplicitPath = kubeconfigPath loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, &clientcmd.ConfigOverrides{}) config, err := loader.RawConfig() if err != nil { return err } if kubeContext != "" { config.CurrentContext = kubeContext } k, err := k8s.NewAPI(kubeconfigPath, config.CurrentContext, impersonate, impersonateGroup, 0) if err != nil { return err } sa, err := k.CoreV1().ServiceAccounts(opts.namespace).Get(cmd.Context(), opts.serviceAccountName, metav1.GetOptions{}) if err != nil { return err } listOpts := metav1.ListOptions{ FieldSelector: fmt.Sprintf("type=%s", corev1.SecretTypeServiceAccountToken), } secrets, err := k.CoreV1().Secrets(opts.namespace).List(cmd.Context(), listOpts) if err != nil { return err } token, err := extractSAToken(secrets.Items, sa.Name) if err != nil { return err } context, ok := config.Contexts[config.CurrentContext] if !ok { return fmt.Errorf("could not extract current context from config") } context.AuthInfo = opts.serviceAccountName config.Contexts = map[string]*api.Context{ config.CurrentContext: context, } config.AuthInfos = map[string]*api.AuthInfo{ opts.serviceAccountName: { Token: token, }, } cluster := config.Clusters[context.Cluster] if opts.apiServerAddress != "" { cluster.Server = opts.apiServerAddress } config.Clusters = map[string]*api.Cluster{ context.Cluster: cluster, } kubeconfig, err := clientcmd.Write(config) if err != nil { return err } creds := corev1.Secret{ Type: k8s.MirrorSecretType, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("cluster-credentials-%s", opts.clusterName), Namespace: opts.namespace, }, Data: map[string][]byte{ k8s.ConfigKeyName: kubeconfig, }, } var credsOut []byte if opts.output == "yaml" { credsOut, err = yaml.Marshal(creds) if err != nil { return err } } else if opts.output == "json" { credsOut, err = json.Marshal(creds) if err != nil { return err } } else { return fmt.Errorf("output format %s not supported", opts.output) } destinationCreds := corev1.Secret{ Type: k8s.MirrorSecretType, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("cluster-credentials-%s", opts.clusterName), Namespace: controlPlaneNamespace, Labels: map[string]string{ clusterNameLabel: opts.clusterName, }, Annotations: map[string]string{ trustDomainAnnotation: configMap.IdentityTrustDomain, clusterDomainAnnotation: configMap.ClusterDomain, }, }, Data: map[string][]byte{ k8s.ConfigKeyName: kubeconfig, }, } var destinationCredsOut []byte if opts.output == "yaml" { destinationCredsOut, err = yaml.Marshal(destinationCreds) if err != nil { return err } } else if opts.output == "json" { destinationCredsOut, err = json.Marshal(destinationCreds) if err != nil { return err } } else { return fmt.Errorf("output format %s not supported", opts.output) } remoteDiscoverySelector, err := metav1.ParseToLabelSelector(opts.remoteDiscoverySelector) if err != nil { return err } federatedServiceSelector, err := metav1.ParseToLabelSelector(opts.federatedServiceSelector) if err != nil { return err } link := v1alpha3.Link{ TypeMeta: metav1.TypeMeta{Kind: "Link", APIVersion: "multicluster.linkerd.io/v1alpha3"}, ObjectMeta: metav1.ObjectMeta{ Name: opts.clusterName, Namespace: opts.namespace, Annotations: map[string]string{ k8s.CreatedByAnnotation: k8s.CreatedByAnnotationValue(), }, }, Spec: v1alpha3.LinkSpec{ TargetClusterName: opts.clusterName, TargetClusterDomain: configMap.ClusterDomain, TargetClusterLinkerdNamespace: controlPlaneNamespace, ClusterCredentialsSecret: fmt.Sprintf("cluster-credentials-%s", opts.clusterName), RemoteDiscoverySelector: remoteDiscoverySelector, FederatedServiceSelector: federatedServiceSelector, }, } // If there is a gateway in the exporting cluster, populate Link // resource with gateway information if opts.enableGateway { gateway, err := k.CoreV1().Services(opts.gatewayNamespace).Get(cmd.Context(), opts.gatewayName, metav1.GetOptions{}) if err != nil { return err } gwAddresses := []string{} for _, ingress := range gateway.Status.LoadBalancer.Ingress { addr := ingress.IP if addr == "" { addr = ingress.Hostname } if addr == "" { continue } gwAddresses = append(gwAddresses, addr) } if opts.gatewayAddresses != "" { link.Spec.GatewayAddress = opts.gatewayAddresses } else if len(gwAddresses) > 0 { link.Spec.GatewayAddress = strings.Join(gwAddresses, ",") } else { return fmt.Errorf("Gateway %s.%s has no ingress addresses", gateway.Name, gateway.Namespace) } gatewayIdentity, ok := gateway.Annotations[k8s.GatewayIdentity] if !ok || gatewayIdentity == "" { return fmt.Errorf("Gateway %s.%s has no %s annotation", gateway.Name, gateway.Namespace, k8s.GatewayIdentity) } link.Spec.GatewayIdentity = gatewayIdentity probeSpec, err := extractProbeSpec(gateway) if err != nil { return err } link.Spec.ProbeSpec = probeSpec gatewayPort, err := extractGatewayPort(gateway) if err != nil { return err } // Override with user provided gateway port if present if opts.gatewayPort != 0 { gatewayPort = opts.gatewayPort } link.Spec.GatewayPort = fmt.Sprintf("%d", gatewayPort) link.Spec.Selector, err = metav1.ParseToLabelSelector(opts.selector) if err != nil { return err } } var linkOut []byte if opts.output == "yaml" { linkOut, err = yaml.Marshal(link) if err != nil { return err } } else if opts.output == "json" { linkOut, err = json.Marshal(link) if err != nil { return err } } else { return fmt.Errorf("output format %s not supported", opts.output) } values, err := buildServiceMirrorValues(opts) if err != nil { return err } // Create values override valuesOverrides, err := valuesOptions.MergeValues(nil) if err != nil { return err } if opts.ha { if valuesOverrides, err = charts.OverrideFromFile( valuesOverrides, static.Templates, helmMulticlusterLinkDefaultChartName, "values-ha.yaml", ); err != nil { return err } } serviceMirrorOut, err := renderServiceMirror(values, valuesOverrides, opts.namespace, opts.output) if err != nil { return err } separator := []byte("---\n") if opts.output == "json" { separator = []byte("\n") } stdout.Write(credsOut) stdout.Write(separator) stdout.Write(destinationCredsOut) stdout.Write(separator) stdout.Write(linkOut) stdout.Write(separator) if opts.enableServiceMirror { stdout.Write(serviceMirrorOut) stdout.Write(separator) } return nil }, } flags.AddValueOptionsFlags(cmd.Flags(), &valuesOptions) cmd.Flags().StringVar(&opts.namespace, "namespace", defaultMulticlusterNamespace, "The namespace for the service account") cmd.Flags().StringVar(&opts.clusterName, "cluster-name", "", "Cluster name") cmd.Flags().StringVar(&opts.apiServerAddress, "api-server-address", "", "The api server address of the target cluster") cmd.Flags().StringVar(&opts.serviceAccountName, "service-account-name", defaultServiceAccountName, "The name of the service account associated with the credentials") cmd.Flags().StringVar(&opts.controlPlaneVersion, "control-plane-version", opts.controlPlaneVersion, "(Development) Tag to be used for the service mirror controller image") cmd.Flags().StringVar(&opts.gatewayName, "gateway-name", defaultGatewayName, "The name of the gateway service") cmd.Flags().StringVar(&opts.gatewayNamespace, "gateway-namespace", defaultMulticlusterNamespace, "The namespace of the gateway service") cmd.Flags().Uint32Var(&opts.serviceMirrorRetryLimit, "service-mirror-retry-limit", opts.serviceMirrorRetryLimit, "The number of times a failed update from the target cluster is allowed to be retried") cmd.Flags().StringVar(&opts.logLevel, "log-level", opts.logLevel, "Log level for the Multicluster components") cmd.Flags().StringVar(&opts.logFormat, "log-format", opts.logFormat, "Log format for the Multicluster components") cmd.Flags().StringVar(&opts.dockerRegistry, "registry", opts.dockerRegistry, fmt.Sprintf("Docker registry to pull service mirror controller image from ($%s)", flags.EnvOverrideDockerRegistry)) cmd.Flags().StringVarP(&opts.selector, "selector", "l", opts.selector, "Selector (label query) to filter which services in the target cluster to mirror") cmd.Flags().StringVar(&opts.remoteDiscoverySelector, "remote-discovery-selector", opts.remoteDiscoverySelector, "Selector (label query) to filter which services in the target cluster to mirror in remote discovery mode") cmd.Flags().StringVar(&opts.federatedServiceSelector, "federated-service-selector", opts.federatedServiceSelector, "Selector (label query) for federated service members in the target cluster") cmd.Flags().StringVar(&opts.gatewayAddresses, "gateway-addresses", opts.gatewayAddresses, "If specified, overwrites gateway addresses when gateway service is not type LoadBalancer (comma separated list)") cmd.Flags().Uint32Var(&opts.gatewayPort, "gateway-port", opts.gatewayPort, "If specified, overwrites gateway port when gateway service is not type LoadBalancer") cmd.Flags().BoolVar(&opts.ha, "ha", opts.ha, "Enable HA configuration for the service-mirror deployment (default false)") cmd.Flags().BoolVar(&opts.enableGateway, "gateway", opts.enableGateway, "If false, allows a link to be created against a cluster that does not have a gateway service") cmd.Flags().BoolVar(&opts.enableServiceMirror, "service-mirror", opts.enableServiceMirror, "If false, only outputs link manifest and credentials secrets") cmd.Flags().StringVarP(&opts.output, "output", "o", "yaml", "Output format. One of: json|yaml") pkgcmd.ConfigureNamespaceFlagCompletion( cmd, []string{"namespace", "gateway-namespace"}, kubeconfigPath, impersonate, impersonateGroup, kubeContext) return cmd } func renderServiceMirror(values *multicluster.Values, valuesOverrides map[string]interface{}, namespace string, format string) ([]byte, error) { files := []*chartloader.BufferedFile{ {Name: chartutil.ChartfileName}, {Name: "templates/service-mirror.yaml"}, {Name: "templates/psp.yaml"}, {Name: "templates/gateway-mirror.yaml"}, } var partialFiles []*chartloader.BufferedFile for _, template := range charts.L5dPartials { partialFiles = append(partialFiles, &chartloader.BufferedFile{Name: template}, ) } // Load all multicluster link chart files into buffer if err := charts.FilesReader(static.Templates, helmMulticlusterLinkDefaultChartName+"/", files); err != nil { return nil, err } // Load all partial chart files into buffer if err := charts.FilesReader(partials.Templates, "", partialFiles); err != nil { return nil, err } // Create a Chart obj from the files chart, err := chartloader.LoadFiles(append(files, partialFiles...)) if err != nil { return nil, err } // Render raw values and create chart config rawValues, err := yaml.Marshal(values) if err != nil { return nil, err } // Store final Values generated from values.yaml and CLI flags err = yaml.Unmarshal(rawValues, &chart.Values) if err != nil { return nil, err } vals, err := chartutil.CoalesceValues(chart, valuesOverrides) if err != nil { return nil, err } fullValues := map[string]interface{}{ "Values": vals, "Release": map[string]interface{}{ "Namespace": namespace, "Service": "CLI", }, } // Attach the final values into the `Values` field for rendering to work renderedTemplates, err := engine.Render(chart, fullValues) if err != nil { return nil, err } // Merge templates and inject var yamlBytes bytes.Buffer for _, tmpl := range chart.Templates { t := path.Join(chart.Metadata.Name, tmpl.Name) if _, err := yamlBytes.WriteString(renderedTemplates[t]); err != nil { return nil, err } } var out bytes.Buffer err = pkgcmd.RenderYAMLAs(&yamlBytes, &out, format) if err != nil { return nil, err } return out.Bytes(), nil } func newLinkOptionsWithDefault() (*linkOptions, error) { defaults, err := multicluster.NewLinkValues() if err != nil { return nil, err } return &linkOptions{ controlPlaneVersion: version.Version, namespace: defaultMulticlusterNamespace, dockerRegistry: pkgcmd.DefaultDockerRegistry, serviceMirrorRetryLimit: defaults.ServiceMirrorRetryLimit, logLevel: defaults.LogLevel, logFormat: defaults.LogFormat, selector: fmt.Sprintf("%s=%s", k8s.DefaultExportedServiceSelector, "true"), remoteDiscoverySelector: fmt.Sprintf("%s=%s", k8s.DefaultExportedServiceSelector, "remote-discovery"), federatedServiceSelector: fmt.Sprintf("%s=%s", k8s.DefaultFederatedServiceSelector, "member"), gatewayAddresses: "", gatewayPort: 0, ha: false, enableGateway: true, enableServiceMirror: true, }, nil } func buildServiceMirrorValues(opts *linkOptions) (*multicluster.Values, error) { if !alphaNumDashDot.MatchString(opts.controlPlaneVersion) { return nil, fmt.Errorf("%s is not a valid version", opts.controlPlaneVersion) } if opts.namespace == "" { return nil, errors.New("you need to specify a namespace") } if _, err := log.ParseLevel(opts.logLevel); err != nil { return nil, fmt.Errorf("--log-level must be one of: panic, fatal, error, warn, info, debug, trace") } if opts.logFormat != "plain" && opts.logFormat != "json" { return nil, fmt.Errorf("--log-format must be one of: plain, json") } if opts.selector != "" && opts.selector != fmt.Sprintf("%s=%s", k8s.DefaultExportedServiceSelector, "true") { if !opts.enableGateway { return nil, fmt.Errorf("--selector and --gateway=false are mutually exclusive") } } if opts.gatewayAddresses != "" && !opts.enableGateway { return nil, fmt.Errorf("--gateway-addresses and --gateway=false are mutually exclusive") } if opts.gatewayPort != 0 && !opts.enableGateway { return nil, fmt.Errorf("--gateway-port and --gateway=false are mutually exclusive") } defaults, err := multicluster.NewLinkValues() if err != nil { return nil, err } defaults.Gateway.Enabled = opts.enableGateway defaults.TargetClusterName = opts.clusterName defaults.ServiceMirrorRetryLimit = opts.serviceMirrorRetryLimit defaults.LogLevel = opts.logLevel defaults.LogFormat = opts.logFormat defaults.ControllerImageVersion = opts.controlPlaneVersion defaults.ControllerImage = fmt.Sprintf("%s/controller", opts.dockerRegistry) return defaults, nil } func extractGatewayPort(gateway *corev1.Service) (uint32, error) { for _, port := range gateway.Spec.Ports { if port.Name == k8s.GatewayPortName { if gateway.Spec.Type == "NodePort" { return uint32(port.NodePort), nil } return uint32(port.Port), nil } } return 0, fmt.Errorf("gateway service %s has no gateway port named %s", gateway.Name, k8s.GatewayPortName) } func extractSAToken(secrets []corev1.Secret, saName string) (string, error) { for _, secret := range secrets { boundSA := secret.Annotations[saNameAnnotationKey] if saName == boundSA { token, ok := secret.Data[tokenKey] if !ok { return "", fmt.Errorf("could not find the token data in service account secret %s", secret.Name) } return string(token), nil } } return "", fmt.Errorf("could not find service account token secret for %s", saName) } // ExtractProbeSpec parses the ProbSpec from a gateway service's annotations. // For now we're not including the failureThreshold and timeout fields which // are new since edge-24.9.3, to avoid errors when attempting to apply them in // clusters with an older Link CRD. func extractProbeSpec(gateway *corev1.Service) (v1alpha3.ProbeSpec, error) { path := gateway.Annotations[k8s.GatewayProbePath] if path == "" { return v1alpha3.ProbeSpec{}, errors.New("probe path is empty") } port, err := extractPort(gateway.Spec, k8s.ProbePortName) if err != nil { return v1alpha3.ProbeSpec{}, err } // the `mirror.linkerd.io/probe-period` annotation is initialized with a // default value of "3", but we require a duration-formatted string. So we // perform the conversion, if required. period := gateway.Annotations[k8s.GatewayProbePeriod] if secs, err := strconv.ParseInt(period, 10, 64); err == nil { dur := time.Duration(secs) * time.Second period = dur.String() } else if _, err := time.ParseDuration(period); err != nil { return v1alpha3.ProbeSpec{}, fmt.Errorf("could not parse probe period: %w", err) } return v1alpha3.ProbeSpec{ Path: path, Port: fmt.Sprintf("%d", port), Period: period, }, nil } func extractPort(spec corev1.ServiceSpec, portName string) (uint32, error) { for _, p := range spec.Ports { if p.Name == portName { if spec.Type == "NodePort" { return uint32(p.NodePort), nil } return uint32(p.Port), nil } } return 0, fmt.Errorf("could not find port with name %s", portName) }