package bootstraptoken import ( "context" "crypto/x509" "fmt" "sort" "strings" "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" kubeclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" clientcertutil "k8s.io/client-go/util/cert" bootstrapapi "k8s.io/cluster-bootstrap/token/api" bootstraputil "k8s.io/cluster-bootstrap/token/util" bootstrapsecretutil "k8s.io/cluster-bootstrap/util/secrets" "k8s.io/klog/v2" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/util/lifted/pubkeypin" ) const ( // When a token is matched with 'BootstrapTokenPattern', the size of validated substrings returned by // regexp functions which contains 'Submatch' in their names will be 3. // Submatch 0 is the match of the entire expression, submatch 1 is // the match of the first parenthesized subexpression, and so on. // e.g.: // result := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch("abcdef.1234567890123456") // result == []string{"abcdef.1234567890123456","abcdef","1234567890123456"} // len(result) == 3 validatedSubstringsSize = 3 // DefaultTokenDuration specifies the default amount of time that a bootstrap token will be valid // Default behaviour is 24 hours DefaultTokenDuration = 24 * time.Hour ) var ( // DefaultUsages is the default usages of bootstrap token DefaultUsages = bootstrapapi.KnownTokenUsages // DefaultGroups is the default groups of bootstrap token DefaultGroups = []string{"system:bootstrappers:karmada:default-cluster-token"} ) // BootstrapToken describes one bootstrap token, stored as a Secret in the cluster type BootstrapToken struct { // Token is used for establishing bidirectional trust between clusters and karmada-control-plane. // Used for joining clusters to the karmada-control-plane. Token *Token // Description sets a human-friendly message why this token exists and what it's used // for, so other administrators can know its purpose. // +optional Description string // TTL defines the time to live for this token. Defaults to 24h. // Expires and TTL are mutually exclusive. // +optional TTL *metav1.Duration // Expires specifies the timestamp when this token expires. Defaults to being set // dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive. // +optional Expires *metav1.Time // Usages describes the ways in which this token can be used. Can by default be used // for establishing bidirectional trust, but that can be changed here. // +optional Usages []string // Groups specifies the extra groups that this token will authenticate as when/if // used for authentication // +optional Groups []string } // Token is a token of the format abcdef.abcdef0123456789 that is used // for both validation of the practically of the API server from a joining cluster's point // of view and as an authentication method for the cluster in the bootstrap phase of // "karmadactl join". This token is and should be short-lived type Token struct { ID string Secret string } // GenerateRegisterCommand generate register command that will be printed func GenerateRegisterCommand(kubeConfig, parentCommand, token string, karmadaContext string) (string, error) { klog.V(1).Info("Print register command") // load the kubeconfig file to get the CA certificate and endpoint config, err := clientcmd.LoadFromFile(kubeConfig) if err != nil { return "", fmt.Errorf("failed to load kubeconfig, err: %w", err) } // load the cluster config with the given karmada-context clusterConfig := GetClusterFromKubeConfig(config, karmadaContext) if clusterConfig == nil { return "", fmt.Errorf("failed to get default cluster config") } // load CA certificates from the kubeconfig (either from PEM data or by file path) var caCerts []*x509.Certificate if clusterConfig.CertificateAuthorityData != nil { caCerts, err = clientcertutil.ParseCertsPEM(clusterConfig.CertificateAuthorityData) if err != nil { return "", fmt.Errorf("failed to parse CA certificate from kubeconfig, err: %w", err) } } else if clusterConfig.CertificateAuthority != "" { caCerts, err = clientcertutil.CertsFromFile(clusterConfig.CertificateAuthority) if err != nil { return "", fmt.Errorf("failed to load CA certificate referenced by kubeconfig, err: %w", err) } } else { return "", fmt.Errorf("no CA certificates found in kubeconfig") } // hash all the CA certs and include their public key pins as trusted values publicKeyPins := make([]string, 0, len(caCerts)) for _, caCert := range caCerts { publicKeyPins = append(publicKeyPins, pubkeypin.Hash(caCert)) } return fmt.Sprintf("%s register %s --token %s --discovery-token-ca-cert-hash %s", parentCommand, strings.Replace(clusterConfig.Server, "https://", "", -1), token, strings.Join(publicKeyPins, ",")), nil } // GetClusterFromKubeConfig returns the Cluster of the specified KubeConfig, if karmada-context unset, it will use the current-context func GetClusterFromKubeConfig(config *clientcmdapi.Config, karmadaContext string) *clientcmdapi.Cluster { // If there is an unnamed cluster object, use it if config.Clusters[""] != nil { return config.Clusters[""] } if karmadaContext == "" { karmadaContext = config.CurrentContext } if config.Contexts[karmadaContext] != nil { return config.Clusters[config.Contexts[karmadaContext].Cluster] } return nil } // GenerateRandomBootstrapToken generate random bootstrap token func GenerateRandomBootstrapToken(ttl *metav1.Duration, description string, groups, usages []string) (*BootstrapToken, error) { tokenStr, err := bootstraputil.GenerateBootstrapToken() if err != nil { return nil, fmt.Errorf("couldn't generate random token, err: %w", err) } token, err := NewToken(tokenStr) if err != nil { return nil, err } bt := &BootstrapToken{ Token: token, TTL: ttl, Description: description, Groups: groups, Usages: usages, } return bt, nil } // NewToken converts the given Bootstrap Token as a string // to the Token object used for serialization/deserialization // and internal usage. It also automatically validates that the given token // is of the right format func NewToken(token string) (*Token, error) { substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token) if len(substrs) != validatedSubstringsSize { return nil, fmt.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern) } return &Token{ID: substrs[1], Secret: substrs[2]}, nil } // ConvertBootstrapTokenToSecret converts the given BootstrapToken object to its Secret representation that // may be submitted to the API Server in order to be stored. func ConvertBootstrapTokenToSecret(bt *BootstrapToken) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: bootstraputil.BootstrapTokenSecretName(bt.Token.ID), Namespace: metav1.NamespaceSystem, }, Type: corev1.SecretType(bootstrapapi.SecretTypeBootstrapToken), Data: encodeTokenSecretData(bt, time.Now()), } } // encodeTokenSecretData takes the token discovery object and an optional duration and returns the .Data for the Secret // now is passed in order to be able to used in unit testing func encodeTokenSecretData(token *BootstrapToken, now time.Time) map[string][]byte { data := map[string][]byte{ bootstrapapi.BootstrapTokenIDKey: []byte(token.Token.ID), bootstrapapi.BootstrapTokenSecretKey: []byte(token.Token.Secret), } if len(token.Description) > 0 { data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte(token.Description) } // If for some strange reason both token.TTL and token.Expires would be set // (they are mutually exclusive in validation so this shouldn't be the case), // token.Expires has higher priority, as can be seen in the logic here. if token.Expires != nil { // Format the expiration date accordingly // TODO: This maybe should be a helper function in bootstraputil? expirationString := token.Expires.Time.UTC().Format(time.RFC3339) data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) } else if token.TTL != nil && token.TTL.Duration > 0 { // Only if .Expires is unset, TTL might have an effect // Get the current time, add the specified duration, and format it accordingly expirationString := now.Add(token.TTL.Duration).UTC().Format(time.RFC3339) data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) } for _, usage := range token.Usages { data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true") } if len(token.Groups) > 0 { data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(strings.Join(token.Groups, ",")) } return data } // NewTokenFromIDAndSecret is a wrapper around NewToken // that allows the caller to specify the ID and Secret separately func NewTokenFromIDAndSecret(id, secret string) (*Token, error) { return NewToken(bootstraputil.TokenFromIDAndSecret(id, secret)) } // GetBootstrapTokenFromSecret returns a BootstrapToken object from the given Secret func GetBootstrapTokenFromSecret(secret *corev1.Secret) (*BootstrapToken, error) { // Get the Token ID field from the Secret data tokenID := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenIDKey) if len(tokenID) == 0 { return nil, fmt.Errorf("bootstrap Token Secret has no token-id data: %s", secret.Name) } // Enforce the right naming convention if secret.Name != bootstraputil.BootstrapTokenSecretName(tokenID) { return nil, fmt.Errorf("bootstrap token name is not of the form '%s(token-id)'. Actual: %q. Expected: %q", bootstrapapi.BootstrapTokenSecretPrefix, secret.Name, bootstraputil.BootstrapTokenSecretName(tokenID)) } tokenSecret := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenSecretKey) if len(tokenSecret) == 0 { return nil, fmt.Errorf("bootstrap Token Secret has no token-secret data: %s", secret.Name) } // Create the Token object based on the ID and Secret bts, err := NewTokenFromIDAndSecret(tokenID, tokenSecret) if err != nil { return nil, fmt.Errorf("bootstrap Token Secret is invalid and couldn't be parsed, err: %w", err) } // Get the description (if any) from the Secret description := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenDescriptionKey) // Expiration time is optional, if not specified this implies the token // never expires. secretExpiration := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExpirationKey) var expires *metav1.Time if len(secretExpiration) > 0 { expTime, err := time.Parse(time.RFC3339, secretExpiration) if err != nil { return nil, fmt.Errorf("can't parse expiration time of bootstrap token %q, err: %w", secret.Name, err) } expires = &metav1.Time{Time: expTime} } // Build an usages string slice from the Secret data var usages []string for k, v := range secret.Data { // Skip all fields that don't include this prefix if !strings.HasPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix) { continue } // Skip those that don't have this usage set to true if string(v) != "true" { continue } usages = append(usages, strings.TrimPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix)) } // Only sort the slice if defined if usages != nil { sort.Strings(usages) } // Get the extra groups information from the Secret // It's done this way to make .Groups be nil in case there is no items, rather than an // empty slice or an empty slice with a "" string only var groups []string groupsString := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExtraGroupsKey) g := strings.Split(groupsString, ",") if len(g) > 0 && len(g[0]) > 0 { groups = g } return &BootstrapToken{ Token: bts, Description: description, Expires: expires, Usages: usages, Groups: groups, }, nil } // CreateNewToken tries to create a token and fails if one with the same ID already exists func CreateNewToken(client kubeclient.Interface, token *BootstrapToken) error { return UpdateOrCreateToken(client, true, token) } // UpdateOrCreateToken attempts to update a token with the given ID, or create if it does not already exist. func UpdateOrCreateToken(client kubeclient.Interface, failIfExists bool, token *BootstrapToken) error { secretName := bootstraputil.BootstrapTokenSecretName(token.Token.ID) secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), secretName, metav1.GetOptions{}) if secret != nil && err == nil && failIfExists { return fmt.Errorf("a token with id %q already exists", token.Token.ID) } updatedOrNewSecret := ConvertBootstrapTokenToSecret(token) // Try to create or update the token with an exponential backoff err = TryRunCommand(func() error { if err := cmdutil.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil { return fmt.Errorf("failed to create or update bootstrap token with name %s, err: %w", secretName, err) } return nil }, 5) if err != nil { return err } return nil } // TryRunCommand runs a function a maximum of failureThreshold times, and retries on error. If failureThreshold is hit; the last error is returned func TryRunCommand(f func() error, failureThreshold int) error { backoff := wait.Backoff{ Duration: 5 * time.Second, Factor: 2, // double the timeout for every failure Steps: failureThreshold, } return wait.ExponentialBackoff(backoff, func() (bool, error) { err := f() if err != nil { // Retry until the timeout return false, nil } // The last f() call was a success, return cleanly return true, nil }) }