/* Copyright 2022 The Karmada Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package bootstraptoken import ( "context" "crypto/x509" "fmt" "sort" "strings" "time" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" kubeclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" clientcertutil "k8s.io/client-go/util/cert" bootstrapapi "k8s.io/cluster-bootstrap/token/api" bootstraputil "k8s.io/cluster-bootstrap/token/util" bootstrapsecretutil "k8s.io/cluster-bootstrap/util/secrets" "k8s.io/klog/v2" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/util/lifted/pubkeypin" ) const ( // When a token is matched with 'BootstrapTokenPattern', the size of validated substrings returned by // regexp functions which contains 'Submatch' in their names will be 3. // Submatch 0 is the match of the entire expression, submatch 1 is // the match of the first parenthesized subexpression, and so on. // e.g.: // result := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch("abcdef.1234567890123456") // result == []string{"abcdef.1234567890123456","abcdef","1234567890123456"} // len(result) == 3 validatedSubstringsSize = 3 // DefaultTokenDuration specifies the default amount of time that a bootstrap token will be valid // Default behaviour is 24 hours DefaultTokenDuration = 24 * time.Hour ) var ( // DefaultUsages is the default usages of bootstrap token DefaultUsages = bootstrapapi.KnownTokenUsages // DefaultGroups is the default groups of bootstrap token DefaultGroups = []string{"system:bootstrappers:karmada:default-cluster-token"} ) // BootstrapToken describes one bootstrap token, stored as a Secret in the cluster type BootstrapToken struct { // Token is used for establishing bidirectional trust between clusters and karmada-control-plane. // Used for joining clusters to the karmada-control-plane. Token *Token // Description sets a human-friendly message why this token exists and what it's used // for, so other administrators can know its purpose. // +optional Description string // TTL defines the time to live for this token. Defaults to 24h. // Expires and TTL are mutually exclusive. // +optional TTL *metav1.Duration // Expires specifies the timestamp when this token expires. Defaults to being set // dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive. // +optional Expires *metav1.Time // Usages describes the ways in which this token can be used. Can by default be used // for establishing bidirectional trust, but that can be changed here. // +optional Usages []string // Groups specifies the extra groups that this token will authenticate as when/if // used for authentication // +optional Groups []string } // Token is a token of the format abcdef.abcdef0123456789 that is used // for both validation of the practically of the API server from a joining cluster's point // of view and as an authentication method for the cluster in the bootstrap phase of // "karmadactl join". This token is and should be short-lived type Token struct { ID string Secret string } // GenerateRegisterCommand generate register command that will be printed func GenerateRegisterCommand(kubeConfig, parentCommand, token string, karmadaContext string) (string, error) { klog.V(1).Info("Print register command") // load the kubeconfig file to get the CA certificate and endpoint config, err := clientcmd.LoadFromFile(kubeConfig) if err != nil { return "", fmt.Errorf("failed to load kubeconfig, err: %w", err) } // load the cluster config with the given karmada-context clusterConfig := GetClusterFromKubeConfig(config, karmadaContext) if clusterConfig == nil { return "", fmt.Errorf("failed to get default cluster config") } // load CA certificates from the kubeconfig (either from PEM data or by file path) var caCerts []*x509.Certificate if clusterConfig.CertificateAuthorityData != nil { caCerts, err = clientcertutil.ParseCertsPEM(clusterConfig.CertificateAuthorityData) if err != nil { return "", fmt.Errorf("failed to parse CA certificate from kubeconfig, err: %w", err) } } else if clusterConfig.CertificateAuthority != "" { caCerts, err = clientcertutil.CertsFromFile(clusterConfig.CertificateAuthority) if err != nil { return "", fmt.Errorf("failed to load CA certificate referenced by kubeconfig, err: %w", err) } } else { return "", fmt.Errorf("no CA certificates found in kubeconfig") } // hash all the CA certs and include their public key pins as trusted values publicKeyPins := make([]string, 0, len(caCerts)) for _, caCert := range caCerts { publicKeyPins = append(publicKeyPins, pubkeypin.Hash(caCert)) } return fmt.Sprintf("%s register %s --token %s --discovery-token-ca-cert-hash %s", parentCommand, strings.Replace(clusterConfig.Server, "https://", "", -1), token, strings.Join(publicKeyPins, ",")), nil } // GetClusterFromKubeConfig returns the Cluster of the specified KubeConfig, if karmada-context unset, it will use the current-context func GetClusterFromKubeConfig(config *clientcmdapi.Config, karmadaContext string) *clientcmdapi.Cluster { // If there is an unnamed cluster object, use it if config.Clusters[""] != nil { return config.Clusters[""] } if karmadaContext == "" { karmadaContext = config.CurrentContext } if config.Contexts[karmadaContext] != nil { return config.Clusters[config.Contexts[karmadaContext].Cluster] } return nil } // GenerateRandomBootstrapToken generate random bootstrap token func GenerateRandomBootstrapToken(ttl *metav1.Duration, description string, groups, usages []string) (*BootstrapToken, error) { tokenStr, err := bootstraputil.GenerateBootstrapToken() if err != nil { return nil, fmt.Errorf("couldn't generate random token, err: %w", err) } token, err := NewToken(tokenStr) if err != nil { return nil, err } bt := &BootstrapToken{ Token: token, TTL: ttl, Description: description, Groups: groups, Usages: usages, } return bt, nil } // NewToken converts the given Bootstrap Token as a string // to the Token object used for serialization/deserialization // and internal usage. It also automatically validates that the given token // is of the right format func NewToken(token string) (*Token, error) { substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token) if len(substrs) != validatedSubstringsSize { return nil, fmt.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern) } return &Token{ID: substrs[1], Secret: substrs[2]}, nil } // ConvertBootstrapTokenToSecret converts the given BootstrapToken object to its Secret representation that // may be submitted to the API Server in order to be stored. func ConvertBootstrapTokenToSecret(bt *BootstrapToken) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: bootstraputil.BootstrapTokenSecretName(bt.Token.ID), Namespace: metav1.NamespaceSystem, }, Type: bootstrapapi.SecretTypeBootstrapToken, Data: encodeTokenSecretData(bt, time.Now()), } } // encodeTokenSecretData takes the token discovery object and an optional duration and returns the .Data for the Secret // now is passed in order to be able to used in unit testing func encodeTokenSecretData(token *BootstrapToken, now time.Time) map[string][]byte { data := map[string][]byte{ bootstrapapi.BootstrapTokenIDKey: []byte(token.Token.ID), bootstrapapi.BootstrapTokenSecretKey: []byte(token.Token.Secret), } if len(token.Description) > 0 { data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte(token.Description) } // If for some strange reason both token.TTL and token.Expires would be set // (they are mutually exclusive in validation so this shouldn't be the case), // token.Expires has higher priority, as can be seen in the logic here. if token.Expires != nil { // Format the expiration date accordingly // TODO: This maybe should be a helper function in bootstraputil? expirationString := token.Expires.Time.UTC().Format(time.RFC3339) data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) } else if token.TTL != nil && token.TTL.Duration > 0 { // Only if .Expires is unset, TTL might have an effect // Get the current time, add the specified duration, and format it accordingly expirationString := now.Add(token.TTL.Duration).UTC().Format(time.RFC3339) data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) } for _, usage := range token.Usages { data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true") } if len(token.Groups) > 0 { data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(strings.Join(token.Groups, ",")) } return data } // NewTokenFromIDAndSecret is a wrapper around NewToken // that allows the caller to specify the ID and Secret separately func NewTokenFromIDAndSecret(id, secret string) (*Token, error) { return NewToken(bootstraputil.TokenFromIDAndSecret(id, secret)) } // GetBootstrapTokenFromSecret returns a BootstrapToken object from the given Secret func GetBootstrapTokenFromSecret(secret *corev1.Secret) (*BootstrapToken, error) { // Get the Token ID field from the Secret data tokenID := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenIDKey) if len(tokenID) == 0 { return nil, fmt.Errorf("bootstrap Token Secret has no token-id data: %s", secret.Name) } // Enforce the right naming convention if secret.Name != bootstraputil.BootstrapTokenSecretName(tokenID) { return nil, fmt.Errorf("bootstrap token name is not of the form '%s(token-id)'. Actual: %q. Expected: %q", bootstrapapi.BootstrapTokenSecretPrefix, secret.Name, bootstraputil.BootstrapTokenSecretName(tokenID)) } tokenSecret := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenSecretKey) if len(tokenSecret) == 0 { return nil, fmt.Errorf("bootstrap Token Secret has no token-secret data: %s", secret.Name) } // Create the Token object based on the ID and Secret bts, err := NewTokenFromIDAndSecret(tokenID, tokenSecret) if err != nil { return nil, fmt.Errorf("bootstrap Token Secret is invalid and couldn't be parsed, err: %w", err) } // Get the description (if any) from the Secret description := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenDescriptionKey) // Expiration time is optional, if not specified this implies the token // never expires. secretExpiration := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExpirationKey) var expires *metav1.Time if len(secretExpiration) > 0 { expTime, err := time.Parse(time.RFC3339, secretExpiration) if err != nil { return nil, fmt.Errorf("can't parse expiration time of bootstrap token %q, err: %w", secret.Name, err) } expires = &metav1.Time{Time: expTime} } // Build an usages string slice from the Secret data var usages []string for k, v := range secret.Data { // Skip all fields that don't include this prefix if !strings.HasPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix) { continue } // Skip those that don't have this usage set to true if string(v) != "true" { continue } usages = append(usages, strings.TrimPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix)) } // Only sort the slice if defined if usages != nil { sort.Strings(usages) } // Get the extra groups information from the Secret // It's done this way to make .Groups be nil in case there is no items, rather than an // empty slice or an empty slice with a "" string only var groups []string groupsString := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExtraGroupsKey) g := strings.Split(groupsString, ",") if len(g) > 0 && len(g[0]) > 0 { groups = g } return &BootstrapToken{ Token: bts, Description: description, Expires: expires, Usages: usages, Groups: groups, }, nil } // CreateNewToken tries to create a token and fails if one with the same ID already exists func CreateNewToken(client kubeclient.Interface, token *BootstrapToken) error { return UpdateOrCreateToken(client, true, token) } // UpdateOrCreateToken attempts to update a token with the given ID, or create if it does not already exist. func UpdateOrCreateToken(client kubeclient.Interface, failIfExists bool, token *BootstrapToken) error { secretName := bootstraputil.BootstrapTokenSecretName(token.Token.ID) secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), secretName, metav1.GetOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } if secret != nil && err == nil && failIfExists { return fmt.Errorf("a token with id %q already exists", token.Token.ID) } updatedOrNewSecret := ConvertBootstrapTokenToSecret(token) // Try to create or update the token with an exponential backoff err = TryRunCommand(func() error { if err := cmdutil.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil { return fmt.Errorf("failed to create or update bootstrap token with name %s, err: %w", secretName, err) } return nil }, 5) if err != nil { return err } return nil } // TryRunCommand runs a function a maximum of failureThreshold times, and retries on error. If failureThreshold is hit; the last error is returned func TryRunCommand(f func() error, failureThreshold int) error { backoff := wait.Backoff{ Duration: 5 * time.Second, Factor: 2, // double the timeout for every failure Steps: failureThreshold, } return wait.ExponentialBackoff(backoff, func() (bool, error) { err := f() if err != nil { // Retry until the timeout return false, nil } // The last f() call was a success, return cleanly return true, nil }) }