diff --git a/pkg/karmadactl/karmadactl.go b/pkg/karmadactl/karmadactl.go index f047da3b6..59aadad0c 100644 --- a/pkg/karmadactl/karmadactl.go +++ b/pkg/karmadactl/karmadactl.go @@ -67,6 +67,7 @@ func NewKarmadaCtlCommand(cmdUse, parentCommand string) *cobra.Command { addons.NewCommandAddons(parentCommand), NewCmdJoin(karmadaConfig, parentCommand), NewCmdUnjoin(karmadaConfig, parentCommand), + NewCmdToken(karmadaConfig, parentCommand, ioStreams), }, }, { diff --git a/pkg/karmadactl/token.go b/pkg/karmadactl/token.go new file mode 100644 index 000000000..c64fc4b54 --- /dev/null +++ b/pkg/karmadactl/token.go @@ -0,0 +1,378 @@ +package karmadactl + +import ( + "context" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/genericclioptions" + kubeclient "k8s.io/client-go/kubernetes" + bootstrapapi "k8s.io/cluster-bootstrap/token/api" + bootstraputil "k8s.io/cluster-bootstrap/token/util" + "k8s.io/klog/v2" + "k8s.io/kubectl/pkg/cmd/get" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/karmada-io/karmada/pkg/karmadactl/options" + tokenutil "github.com/karmada-io/karmada/pkg/karmadactl/util/bootstraptoken" +) + +var ( + tokenLong = templates.LongDesc(` + This command manages bootstrap tokens. It is optional and needed only for advanced use cases. + + In short, bootstrap tokens are used for establishing bidirectional trust between a client and a server. + A bootstrap token can be used when a client (for example a member cluster that is about to join control plane) needs + to trust the server it is talking to. Then a bootstrap token with the "signing" usage can be used. + bootstrap tokens can also function as a way to allow short-lived authentication to the API Server + (the token serves as a way for the API Server to trust the client), for example for doing the TLS Bootstrap. + + What is a bootstrap token more exactly? + - It is a Secret in the kube-system namespace of type "bootstrap.kubernetes.io/token". + - A bootstrap token must be of the form "[a-z0-9]{6}.[a-z0-9]{16}". The former part is the public token ID, + while the latter is the Token Secret and it must be kept private at all circumstances! + - The name of the Secret must be named "bootstrap-token-(token-id)". + + This command is same as 'kubeadm token', but it will create tokens that are used by member clusters.`) + + tokenExamples = templates.Examples(` + # Create a token and print the full '%[1]s register' flag needed to join the cluster using the token. + %[1]s token create --print-register-command + `) +) + +// NewCmdToken returns cobra.Command for token management +func NewCmdToken(karmadaConfig KarmadaConfig, parentCommand string, streams genericclioptions.IOStreams) *cobra.Command { + opts := &CommandTokenOptions{ + parentCommand: parentCommand, + TTL: &metav1.Duration{ + Duration: 0, + }, + } + + cmd := &cobra.Command{ + Use: "token", + Short: "Manage bootstrap tokens", + Long: tokenLong, + Example: fmt.Sprintf(tokenExamples, parentCommand), + } + + cmd.AddCommand(NewCmdTokenCreate(karmadaConfig, streams.Out, opts)) + cmd.AddCommand(NewCmdTokenList(karmadaConfig, streams.Out, streams.ErrOut, opts)) + cmd.AddCommand(NewCmdTokenDelete(karmadaConfig, streams.Out, opts)) + + return cmd +} + +// CommandTokenOptions holds all command options for token +type CommandTokenOptions struct { + // global flags + options.GlobalCommandOptions + + TTL *metav1.Duration + Description string + Groups []string + Usages []string + + PrintRegisterCommand bool + + parentCommand string +} + +// NewCmdTokenCreate returns cobra.Command to create token +func NewCmdTokenCreate(karmadaConfig KarmadaConfig, out io.Writer, tokenOpts *CommandTokenOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "create", + DisableFlagsInUseLine: true, + Short: "Create bootstrap tokens on the server", + Long: templates.LongDesc(` + This command will create a bootstrap token for you. + You can specify the usages for this token, the "time to live" and an optional human friendly description. + + This should be a securely generated random token of the form "[a-z0-9]{6}.[a-z0-9]{16}". + `), + RunE: func(Cmd *cobra.Command, args []string) error { + // Get control plane kube-apiserver client + client, err := tokenOpts.getClientSet(karmadaConfig) + if err != nil { + return err + } + + return tokenOpts.runCreateToken(out, client) + }, + Args: cobra.NoArgs, + } + + tokenOpts.GlobalCommandOptions.AddFlags(cmd.Flags()) + + if tokenOpts.KubeConfig == "" { + env := os.Getenv("KUBECONFIG") + if env != "" { + tokenOpts.KubeConfig = env + } else { + tokenOpts.KubeConfig = defaultKubeConfig + } + } + + cmd.Flags().BoolVar(&tokenOpts.PrintRegisterCommand, "print-register-command", false, fmt.Sprintf("Instead of printing only the token, print the full '%s join' flag needed to join the member cluster using the token.", tokenOpts.parentCommand)) + cmd.Flags().DurationVar(&tokenOpts.TTL.Duration, "ttl", tokenutil.DefaultTokenDuration, "The duration before the token is automatically deleted (e.g. 1s, 2m, 3h). If set to '0', the token will never expire") + cmd.Flags().StringSliceVar(&tokenOpts.Usages, "usages", tokenutil.DefaultUsages, fmt.Sprintf("Describes the ways in which this token can be used. You can pass --usages multiple times or provide a comma separated list of options. Valid options: [%s]", strings.Join(bootstrapapi.KnownTokenUsages, ","))) + cmd.Flags().StringSliceVar(&tokenOpts.Groups, "groups", tokenutil.DefaultGroups, fmt.Sprintf("Extra groups that this token will authenticate as when used for authentication. Must match %q", bootstrapapi.BootstrapGroupPattern)) + cmd.Flags().StringVar(&tokenOpts.Description, "description", tokenOpts.Description, "A human friendly description of how this token is used.") + + return cmd +} + +// NewCmdTokenList returns cobra.Command to list tokens +func NewCmdTokenList(karmadaConfig KarmadaConfig, out io.Writer, errW io.Writer, tokenOpts *CommandTokenOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List bootstrap tokens on the server", + Long: "This command will list all bootstrap tokens for you.", + RunE: func(tokenCmd *cobra.Command, args []string) error { + // Get control plane kube-apiserver client + client, err := tokenOpts.getClientSet(karmadaConfig) + if err != nil { + return err + } + + return tokenOpts.runListTokens(client, out, errW) + }, + Args: cobra.NoArgs, + } + + tokenOpts.GlobalCommandOptions.AddFlags(cmd.Flags()) + + if tokenOpts.KubeConfig == "" { + env := os.Getenv("KUBECONFIG") + if env != "" { + tokenOpts.KubeConfig = env + } else { + tokenOpts.KubeConfig = defaultKubeConfig + } + } + + return cmd +} + +// NewCmdTokenDelete returns cobra.Command to delete tokens +func NewCmdTokenDelete(karmadaConfig KarmadaConfig, out io.Writer, tokenOpts *CommandTokenOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "delete [token-value] ...", + DisableFlagsInUseLine: true, + Short: "Delete bootstrap tokens on the server", + Long: templates.LongDesc(` + This command will delete a list of bootstrap tokens for you. + + The [token-value] is the full Token of the form "[a-z0-9]{6}.[a-z0-9]{16}" or the + Token ID of the form "[a-z0-9]{6}" to delete. + `), + RunE: func(tokenCmd *cobra.Command, args []string) error { + if len(args) < 1 { + return fmt.Errorf("missing subcommand; 'token delete' is missing token of form %q", bootstrapapi.BootstrapTokenIDPattern) + } + + // Get control plane kube-apiserver client + client, err := tokenOpts.getClientSet(karmadaConfig) + if err != nil { + return err + } + + return tokenOpts.runDeleteTokens(out, client, args) + }, + } + + tokenOpts.GlobalCommandOptions.AddFlags(cmd.Flags()) + + if tokenOpts.KubeConfig == "" { + env := os.Getenv("KUBECONFIG") + if env != "" { + tokenOpts.KubeConfig = env + } else { + tokenOpts.KubeConfig = defaultKubeConfig + } + } + + return cmd +} + +// runCreateToken generates a new bootstrap token and stores it as a secret on the server. +func (o *CommandTokenOptions) runCreateToken(out io.Writer, client kubeclient.Interface) error { + klog.V(1).Infoln("[token] creating token") + bootstrapToken, err := tokenutil.GenerateRandomBootstrapToken(o.TTL, o.Description, o.Groups, o.Usages) + if err != nil { + return err + } + + if err := tokenutil.CreateNewToken(client, bootstrapToken); err != nil { + return err + } + + tokenStr := bootstrapToken.Token.ID + "." + bootstrapToken.Token.Secret + + // if --print-register-command was specified, print a machine-readable full `karmadactl register` command + // otherwise, just print the token + if o.PrintRegisterCommand { + joinCommand, err := tokenutil.GenerateRegisterCommand(o.KubeConfig, o.parentCommand, tokenStr) + if err != nil { + return fmt.Errorf("failed to get register command, err: %w", err) + } + fmt.Fprintln(out, joinCommand) + } else { + fmt.Fprintln(out, tokenStr) + } + + return nil +} + +// runListTokens lists details on all existing bootstrap tokens on the server. +func (o *CommandTokenOptions) runListTokens(client kubeclient.Interface, out io.Writer, errW io.Writer) error { + // First, build our selector for bootstrap tokens only + klog.V(1).Infoln("[token] preparing selector for bootstrap token") + tokenSelector := fields.SelectorFromSet( + map[string]string{ + "type": string(bootstrapapi.SecretTypeBootstrapToken), + }, + ) + listOptions := metav1.ListOptions{ + FieldSelector: tokenSelector.String(), + } + + klog.V(1).Info("[token] retrieving list of bootstrap tokens") + secrets, err := client.CoreV1().Secrets(metav1.NamespaceSystem).List(context.TODO(), listOptions) + if err != nil { + return fmt.Errorf("failed to list bootstrap tokens, err: %w", err) + } + + printFlags := get.NewGetPrintFlags() + + printFlags.SetKind(schema.GroupKind{Group: "output.karmada.io", Kind: "BootstrapToken"}) + + printer, err := printFlags.ToPrinter() + if err != nil { + return err + } + + // only print token with table format + printer = &get.TablePrinter{Delegate: printer} + + tokenTable := &metav1.Table{} + setColumnDefinition(tokenTable) + + for idx := range secrets.Items { + // Get the BootstrapToken struct representation from the Secret object + token, err := tokenutil.GetBootstrapTokenFromSecret(&secrets.Items[idx]) + if err != nil { + fmt.Fprintf(errW, "%v", err) + continue + } + + outputToken := tokenutil.BootstrapToken{ + Token: &tokenutil.Token{ID: token.Token.ID, Secret: token.Token.Secret}, + Description: token.Description, + TTL: token.TTL, + Expires: token.Expires, + Usages: token.Usages, + Groups: token.Groups, + } + + tokenTable.Rows = append(tokenTable.Rows, constructTokenTableRow(outputToken)) + } + + if err := printer.PrintObj(tokenTable, out); err != nil { + return fmt.Errorf("unable to print tokens, err: %w", err) + } + + return nil +} + +// runDeleteTokens removes a bootstrap tokens from the server. +func (o *CommandTokenOptions) runDeleteTokens(out io.Writer, client kubeclient.Interface, tokenIDsOrTokens []string) error { + for _, tokenIDOrToken := range tokenIDsOrTokens { + // Assume this is a token id and try to parse it + tokenID := tokenIDOrToken + klog.V(1).Info("[token] parsing token") + if !bootstraputil.IsValidBootstrapTokenID(tokenIDOrToken) { + // Okay, the full token with both id and secret was probably passed. Parse it and extract the ID only + bts, err := tokenutil.NewToken(tokenIDOrToken) + if err != nil { + return fmt.Errorf("given token didn't match pattern %q or %q", + bootstrapapi.BootstrapTokenIDPattern, bootstrapapi.BootstrapTokenIDPattern) + } + tokenID = bts.ID + } + + tokenSecretName := bootstraputil.BootstrapTokenSecretName(tokenID) + klog.V(1).Infof("[token] deleting token %q", tokenID) + if err := client.CoreV1().Secrets(metav1.NamespaceSystem).Delete(context.TODO(), tokenSecretName, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete bootstrap token %q, err: %w", tokenID, err) + } + fmt.Fprintf(out, "bootstrap token %q deleted\n", tokenID) + } + return nil +} + +// getClientSet get clientset of karmada control plane +func (o *CommandTokenOptions) getClientSet(karmadaConfig KarmadaConfig) (kubeclient.Interface, error) { + // Get control plane karmada-apiserver client + controlPlaneRestConfig, err := karmadaConfig.GetRestConfig(o.KarmadaContext, o.KubeConfig) + if err != nil { + return nil, fmt.Errorf("failed to get control plane rest config. context: %s, kube-config: %s, error: %v", + o.KarmadaContext, o.KubeConfig, err) + } + return kubeclient.NewForConfigOrDie(controlPlaneRestConfig), nil +} + +// constructTokenTableRow construct token table row +func constructTokenTableRow(token tokenutil.BootstrapToken) metav1.TableRow { + var row metav1.TableRow + + tokenString := token.Token.ID + "." + token.Token.Secret + + ttl := "" + expires := "" + if token.Expires != nil { + ttl = duration.ShortHumanDuration(time.Until(token.Expires.Time)) + expires = token.Expires.Format(time.RFC3339) + } + + usages := strings.Join(token.Usages, ",") + if len(usages) == 0 { + usages = "" + } + + description := token.Description + if len(description) == 0 { + description = "" + } + + groups := strings.Join(token.Groups, ",") + if len(groups) == 0 { + groups = "" + } + + row.Cells = append(row.Cells, tokenString, ttl, expires, usages, description, groups) + + return row +} + +// setColumnDefinition set print ColumnDefinition +func setColumnDefinition(table *metav1.Table) { + tokenColumns := []metav1.TableColumnDefinition{ + {Name: "TOKEN", Type: "string", Format: "", Priority: 0}, + {Name: "TTL", Type: "string", Format: "", Priority: 0}, + {Name: "EXPIRES", Type: "string", Format: "", Priority: 0}, + {Name: "USAGES", Type: "string", Format: "", Priority: 0}, + {Name: "DESCRIPTION", Type: "string", Format: "", Priority: 0}, + {Name: "EXTRA GROUPS", Type: "string", Format: "", Priority: 0}, + } + table.ColumnDefinitions = tokenColumns +} diff --git a/pkg/karmadactl/util/bootstraptoken/bootstraptoken.go b/pkg/karmadactl/util/bootstraptoken/bootstraptoken.go new file mode 100644 index 000000000..c89e1c694 --- /dev/null +++ b/pkg/karmadactl/util/bootstraptoken/bootstraptoken.go @@ -0,0 +1,353 @@ +package bootstraptoken + +import ( + "context" + "crypto/x509" + "fmt" + "sort" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + clientcertutil "k8s.io/client-go/util/cert" + bootstrapapi "k8s.io/cluster-bootstrap/token/api" + bootstraputil "k8s.io/cluster-bootstrap/token/util" + bootstrapsecretutil "k8s.io/cluster-bootstrap/util/secrets" + "k8s.io/klog/v2" + + karmadautil "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/lifted/pubkeypin" +) + +const ( + // When a token is matched with 'BootstrapTokenPattern', the size of validated substrings returned by + // regexp functions which contains 'Submatch' in their names will be 3. + // Submatch 0 is the match of the entire expression, submatch 1 is + // the match of the first parenthesized subexpression, and so on. + // e.g.: + // result := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch("abcdef.1234567890123456") + // result == []string{"abcdef.1234567890123456","abcdef","1234567890123456"} + // len(result) == 3 + validatedSubstringsSize = 3 + + // DefaultTokenDuration specifies the default amount of time that a bootstrap token will be valid + // Default behaviour is 24 hours + DefaultTokenDuration = 24 * time.Hour +) + +var ( + // DefaultUsages is the default usages of bootstrap token + DefaultUsages = bootstrapapi.KnownTokenUsages + // DefaultGroups is the default groups of bootstrap token + DefaultGroups = []string{"system:bootstrappers:karmada:default-cluster-token"} +) + +// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster +type BootstrapToken struct { + // Token is used for establishing bidirectional trust between clusters and karmada-control-plane. + // Used for joining clusters to the karmada-control-plane. + Token *Token + // Description sets a human-friendly message why this token exists and what it's used + // for, so other administrators can know its purpose. + // +optional + Description string + // TTL defines the time to live for this token. Defaults to 24h. + // Expires and TTL are mutually exclusive. + // +optional + TTL *metav1.Duration + // Expires specifies the timestamp when this token expires. Defaults to being set + // dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive. + // +optional + Expires *metav1.Time + // Usages describes the ways in which this token can be used. Can by default be used + // for establishing bidirectional trust, but that can be changed here. + // +optional + Usages []string + // Groups specifies the extra groups that this token will authenticate as when/if + // used for authentication + // +optional + Groups []string +} + +// Token is a token of the format abcdef.abcdef0123456789 that is used +// for both validation of the practically of the API server from a joining cluster's point +// of view and as an authentication method for the cluster in the bootstrap phase of +// "karmadactl join". This token is and should be short-lived +type Token struct { + ID string + Secret string +} + +// GenerateRegisterCommand generate register command that will be printed +func GenerateRegisterCommand(kubeConfig, parentCommand, token string) (string, error) { + klog.V(1).Info("print register command") + // load the kubeconfig file to get the CA certificate and endpoint + config, err := clientcmd.LoadFromFile(kubeConfig) + if err != nil { + return "", fmt.Errorf("failed to load kubeconfig, err: %w", err) + } + + // load the default cluster config + clusterConfig := GetClusterFromKubeConfig(config) + if clusterConfig == nil { + return "", fmt.Errorf("failed to get default cluster config") + } + + // load CA certificates from the kubeconfig (either from PEM data or by file path) + var caCerts []*x509.Certificate + if clusterConfig.CertificateAuthorityData != nil { + caCerts, err = clientcertutil.ParseCertsPEM(clusterConfig.CertificateAuthorityData) + if err != nil { + return "", fmt.Errorf("failed to parse CA certificate from kubeconfig, err: %w", err) + } + } else if clusterConfig.CertificateAuthority != "" { + caCerts, err = clientcertutil.CertsFromFile(clusterConfig.CertificateAuthority) + if err != nil { + return "", fmt.Errorf("failed to load CA certificate referenced by kubeconfig, err: %w", err) + } + } else { + return "", fmt.Errorf("no CA certificates found in kubeconfig") + } + + // hash all the CA certs and include their public key pins as trusted values + publicKeyPins := make([]string, 0, len(caCerts)) + for _, caCert := range caCerts { + publicKeyPins = append(publicKeyPins, pubkeypin.Hash(caCert)) + } + + return fmt.Sprintf("%s register %s --token %s --discovery-token-ca-cert-hash %s", + parentCommand, strings.Replace(clusterConfig.Server, "https://", "", -1), + token, strings.Join(publicKeyPins, ",")), nil +} + +// GetClusterFromKubeConfig returns the default Cluster of the specified KubeConfig +func GetClusterFromKubeConfig(config *clientcmdapi.Config) *clientcmdapi.Cluster { + // If there is an unnamed cluster object, use it + if config.Clusters[""] != nil { + return config.Clusters[""] + } + if config.Contexts[config.CurrentContext] != nil { + return config.Clusters[config.Contexts[config.CurrentContext].Cluster] + } + return nil +} + +// GenerateRandomBootstrapToken generate random bootstrap token +func GenerateRandomBootstrapToken(ttl *metav1.Duration, description string, groups, usages []string) (*BootstrapToken, error) { + tokenStr, err := bootstraputil.GenerateBootstrapToken() + if err != nil { + return nil, fmt.Errorf("couldn't generate random token, err: %w", err) + } + + token, err := NewToken(tokenStr) + if err != nil { + return nil, err + } + + bt := &BootstrapToken{ + Token: token, + TTL: ttl, + Description: description, + Groups: groups, + Usages: usages, + } + + return bt, nil +} + +// NewToken converts the given Bootstrap Token as a string +// to the Token object used for serialization/deserialization +// and internal usage. It also automatically validates that the given token +// is of the right format +func NewToken(token string) (*Token, error) { + substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token) + if len(substrs) != validatedSubstringsSize { + return nil, fmt.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern) + } + + return &Token{ID: substrs[1], Secret: substrs[2]}, nil +} + +// ConvertBootstrapTokenToSecret converts the given BootstrapToken object to its Secret representation that +// may be submitted to the API Server in order to be stored. +func ConvertBootstrapTokenToSecret(bt *BootstrapToken) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: bootstraputil.BootstrapTokenSecretName(bt.Token.ID), + Namespace: metav1.NamespaceSystem, + }, + Type: corev1.SecretType(bootstrapapi.SecretTypeBootstrapToken), + Data: encodeTokenSecretData(bt, time.Now()), + } +} + +// encodeTokenSecretData takes the token discovery object and an optional duration and returns the .Data for the Secret +// now is passed in order to be able to used in unit testing +func encodeTokenSecretData(token *BootstrapToken, now time.Time) map[string][]byte { + data := map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(token.Token.ID), + bootstrapapi.BootstrapTokenSecretKey: []byte(token.Token.Secret), + } + + if len(token.Description) > 0 { + data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte(token.Description) + } + + // If for some strange reason both token.TTL and token.Expires would be set + // (they are mutually exclusive in validation so this shouldn't be the case), + // token.Expires has higher priority, as can be seen in the logic here. + if token.Expires != nil { + // Format the expiration date accordingly + // TODO: This maybe should be a helper function in bootstraputil? + expirationString := token.Expires.Time.UTC().Format(time.RFC3339) + data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) + } else if token.TTL != nil && token.TTL.Duration > 0 { + // Only if .Expires is unset, TTL might have an effect + // Get the current time, add the specified duration, and format it accordingly + expirationString := now.Add(token.TTL.Duration).UTC().Format(time.RFC3339) + data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) + } + + for _, usage := range token.Usages { + data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true") + } + + if len(token.Groups) > 0 { + data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(strings.Join(token.Groups, ",")) + } + return data +} + +// NewTokenFromIDAndSecret is a wrapper around NewToken +// that allows the caller to specify the ID and Secret separately +func NewTokenFromIDAndSecret(id, secret string) (*Token, error) { + return NewToken(bootstraputil.TokenFromIDAndSecret(id, secret)) +} + +// GetBootstrapTokenFromSecret returns a BootstrapToken object from the given Secret +func GetBootstrapTokenFromSecret(secret *corev1.Secret) (*BootstrapToken, error) { + // Get the Token ID field from the Secret data + tokenID := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenIDKey) + if len(tokenID) == 0 { + return nil, fmt.Errorf("bootstrap Token Secret has no token-id data: %s", secret.Name) + } + + // Enforce the right naming convention + if secret.Name != bootstraputil.BootstrapTokenSecretName(tokenID) { + return nil, fmt.Errorf("bootstrap token name is not of the form '%s(token-id)'. Actual: %q. Expected: %q", + bootstrapapi.BootstrapTokenSecretPrefix, secret.Name, bootstraputil.BootstrapTokenSecretName(tokenID)) + } + + tokenSecret := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenSecretKey) + if len(tokenSecret) == 0 { + return nil, fmt.Errorf("bootstrap Token Secret has no token-secret data: %s", secret.Name) + } + + // Create the Token object based on the ID and Secret + bts, err := NewTokenFromIDAndSecret(tokenID, tokenSecret) + if err != nil { + return nil, fmt.Errorf("bootstrap Token Secret is invalid and couldn't be parsed, err: %w", err) + } + + // Get the description (if any) from the Secret + description := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenDescriptionKey) + + // Expiration time is optional, if not specified this implies the token + // never expires. + secretExpiration := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExpirationKey) + var expires *metav1.Time + if len(secretExpiration) > 0 { + expTime, err := time.Parse(time.RFC3339, secretExpiration) + if err != nil { + return nil, fmt.Errorf("can't parse expiration time of bootstrap token %q, err: %w", secret.Name, err) + } + expires = &metav1.Time{Time: expTime} + } + + // Build an usages string slice from the Secret data + var usages []string + for k, v := range secret.Data { + // Skip all fields that don't include this prefix + if !strings.HasPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix) { + continue + } + // Skip those that don't have this usage set to true + if string(v) != "true" { + continue + } + usages = append(usages, strings.TrimPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix)) + } + // Only sort the slice if defined + if usages != nil { + sort.Strings(usages) + } + + // Get the extra groups information from the Secret + // It's done this way to make .Groups be nil in case there is no items, rather than an + // empty slice or an empty slice with a "" string only + var groups []string + groupsString := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExtraGroupsKey) + g := strings.Split(groupsString, ",") + if len(g) > 0 && len(g[0]) > 0 { + groups = g + } + + return &BootstrapToken{ + Token: bts, + Description: description, + Expires: expires, + Usages: usages, + Groups: groups, + }, nil +} + +// CreateNewToken tries to create a token and fails if one with the same ID already exists +func CreateNewToken(client kubeclient.Interface, token *BootstrapToken) error { + return UpdateOrCreateToken(client, true, token) +} + +// UpdateOrCreateToken attempts to update a token with the given ID, or create if it does not already exist. +func UpdateOrCreateToken(client kubeclient.Interface, failIfExists bool, token *BootstrapToken) error { + secretName := bootstraputil.BootstrapTokenSecretName(token.Token.ID) + secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), secretName, metav1.GetOptions{}) + if secret != nil && err == nil && failIfExists { + return fmt.Errorf("a token with id %q already exists", token.Token.ID) + } + + updatedOrNewSecret := ConvertBootstrapTokenToSecret(token) + // Try to create or update the token with an exponential backoff + err = TryRunCommand(func() error { + if err := karmadautil.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil { + return fmt.Errorf("failed to create or update bootstrap token with name %s, err: %w", secretName, err) + } + return nil + }, 5) + if err != nil { + return err + } + + return nil +} + +// TryRunCommand runs a function a maximum of failureThreshold times, and retries on error. If failureThreshold is hit; the last error is returned +func TryRunCommand(f func() error, failureThreshold int) error { + backoff := wait.Backoff{ + Duration: 5 * time.Second, + Factor: 2, // double the timeout for every failure + Steps: failureThreshold, + } + return wait.ExponentialBackoff(backoff, func() (bool, error) { + err := f() + if err != nil { + // Retry until the timeout + return false, nil + } + // The last f() call was a success, return cleanly + return true, nil + }) +} diff --git a/pkg/util/lifted/pubkeypin/pubkeypin.go b/pkg/util/lifted/pubkeypin/pubkeypin.go new file mode 100644 index 000000000..b4c8a6088 --- /dev/null +++ b/pkg/util/lifted/pubkeypin/pubkeypin.go @@ -0,0 +1,127 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package pubkeypin provides primitives for x509 public key pinning in the +// style of RFC7469. + +// This code is directly lifted from the Kubernetes codebase in order to avoid relying on the k8s.io/kubernetes package. +// For reference: +// https://github.com/kubernetes/kubernetes/blob/release-1.24/cmd/kubeadm/app/util/pubkeypin/pubkeypin.go + +package pubkeypin + +import ( + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "fmt" + "strings" +) + +const ( + // formatSHA256 is the prefix for pins that are full-length SHA-256 hashes encoded in base 16 (hex) + formatSHA256 = "sha256" +) + +var ( + // supportedFormats enumerates the supported formats + supportedFormats = strings.Join([]string{formatSHA256}, ", ") +) + +// Set is a set of pinned x509 public keys. +type Set struct { + sha256Hashes map[string]bool +} + +// NewSet returns a new, empty PubKeyPinSet +func NewSet() *Set { + return &Set{make(map[string]bool)} +} + +// Allow adds an allowed public key hash to the Set +func (s *Set) Allow(pubKeyHashes ...string) error { + for _, pubKeyHash := range pubKeyHashes { + parts := strings.Split(pubKeyHash, ":") + if len(parts) != 2 { + return fmt.Errorf("invalid hash, expected \"format:hex-value\". "+ + "Known format(s) are: %s", supportedFormats) + } + format, value := parts[0], parts[1] + + switch strings.ToLower(format) { + case "sha256": + if err := s.allowSHA256(value); err != nil { + return fmt.Errorf("invalid hash %q, %v", pubKeyHash, err) + } + default: + return fmt.Errorf("unknown hash format %q. Known format(s) are: %s", format, supportedFormats) + } + } + return nil +} + +// CheckAny checks if at least one certificate matches one of the public keys in the set +func (s *Set) CheckAny(certificates []*x509.Certificate) error { + var hashes []string + + for _, certificate := range certificates { + if s.checkSHA256(certificate) { + return nil + } + + hashes = append(hashes, Hash(certificate)) + } + return fmt.Errorf("none of the public keys %q are pinned", strings.Join(hashes, ":")) +} + +// Empty returns true if the Set contains no pinned public keys. +func (s *Set) Empty() bool { + return len(s.sha256Hashes) == 0 +} + +// Hash calculates the SHA-256 hash of the Subject Public Key Information (SPKI) +// object in an x509 certificate (in DER encoding). It returns the full hash as a +// hex encoded string (suitable for passing to Set.Allow). +func Hash(certificate *x509.Certificate) string { + spkiHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) + return formatSHA256 + ":" + strings.ToLower(hex.EncodeToString(spkiHash[:])) +} + +// allowSHA256 validates a "sha256" format hash and adds a canonical version of it into the Set +func (s *Set) allowSHA256(hash string) error { + // validate that the hash is the right length to be a full SHA-256 hash + hashLength := hex.DecodedLen(len(hash)) + if hashLength != sha256.Size { + return fmt.Errorf("expected a %d byte SHA-256 hash, found %d bytes", sha256.Size, hashLength) + } + + // validate that the hash is valid hex + _, err := hex.DecodeString(hash) + if err != nil { + return fmt.Errorf("could not decode SHA-256 from hex, err: %w", err) + } + + // in the end, just store the original hex string in memory (in lowercase) + s.sha256Hashes[strings.ToLower(hash)] = true + return nil +} + +// checkSHA256 returns true if the certificate's "sha256" hash is pinned in the Set +func (s *Set) checkSHA256(certificate *x509.Certificate) bool { + actualHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) + actualHashHex := strings.ToLower(hex.EncodeToString(actualHash[:])) + return s.sha256Hashes[actualHashHex] +} diff --git a/pkg/util/lifted/pubkeypin/pubkeypin_test.go b/pkg/util/lifted/pubkeypin/pubkeypin_test.go new file mode 100644 index 000000000..af726ea9b --- /dev/null +++ b/pkg/util/lifted/pubkeypin/pubkeypin_test.go @@ -0,0 +1,168 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pubkeypin + +import ( + "crypto/x509" + "encoding/pem" + "strings" + "testing" +) + +// testCertPEM is a simple self-signed test certificate issued with the openssl CLI: +// openssl req -new -newkey rsa:2048 -days 36500 -nodes -x509 -keyout /dev/null -out test.crt +const testCertPEM = ` +-----BEGIN CERTIFICATE----- +MIIDRDCCAiygAwIBAgIJAJgVaCXvC6HkMA0GCSqGSIb3DQEBBQUAMB8xHTAbBgNV +BAMTFGt1YmVhZG0ta2V5cGlucy10ZXN0MCAXDTE3MDcwNTE3NDMxMFoYDzIxMTcw +NjExMTc0MzEwWjAfMR0wGwYDVQQDExRrdWJlYWRtLWtleXBpbnMtdGVzdDCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK0ba8mHU9UtYlzM1Own2Fk/XGjR +J4uJQvSeGLtz1hID1IA0dLwruvgLCPadXEOw/f/IWIWcmT+ZmvIHZKa/woq2iHi5 ++HLhXs7aG4tjKGLYhag1hLjBI7icqV7ovkjdGAt9pWkxEzhIYClFMXDjKpMSynu+ +YX6nZ9tic1cOkHmx2yiZdMkuriRQnpTOa7bb03OC1VfGl7gHlOAIYaj4539WCOr8 ++ACTUMJUFEHcRZ2o8a/v6F9GMK+7SC8SJUI+GuroXqlMAdhEv4lX5Co52enYaClN ++D9FJLRpBv2YfiCQdJRaiTvCBSxEFz6BN+PtP5l2Hs703ZWEkOqCByM6HV8CAwEA +AaOBgDB+MB0GA1UdDgQWBBRQgUX8MhK2rWBWQiPHWcKzoWDH5DBPBgNVHSMESDBG +gBRQgUX8MhK2rWBWQiPHWcKzoWDH5KEjpCEwHzEdMBsGA1UEAxMUa3ViZWFkbS1r +ZXlwaW5zLXRlc3SCCQCYFWgl7wuh5DAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEB +BQUAA4IBAQCaAUif7Pfx3X0F08cxhx8/Hdx4jcJw6MCq6iq6rsXM32ge43t8OHKC +pJW08dk58a3O1YQSMMvD6GJDAiAfXzfwcwY6j258b1ZlI9Ag0VokvhMl/XfdCsdh +AWImnL1t4hvU5jLaImUUMlYxMcSfHBGAm7WJIZ2LdEfg6YWfZh+WGbg1W7uxLxk6 +y4h5rWdNnzBHWAGf7zJ0oEDV6W6RSwNXtC0JNnLaeIUm/6xdSddJlQPwUv8YH4jX +c1vuFqTnJBPcb7W//R/GI2Paicm1cmns9NLnPR35exHxFTy+D1yxmGokpoPMdife +aH+sfuxT8xeTPb3kjzF9eJTlnEquUDLM +-----END CERTIFICATE-----` + +// expectedHash can be verified using the openssl CLI. +const expectedHash = `sha256:345959acb2c3b2feb87d281961c893f62a314207ef02599f1cc4a5fb255480b3` + +// testCert2PEM is a second test cert generated the same way as testCertPEM +const testCert2PEM = ` +-----BEGIN CERTIFICATE----- +MIID9jCCAt6gAwIBAgIJAN5MXZDic7qYMA0GCSqGSIb3DQEBBQUAMFkxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCXRlc3RDZXJ0MjAgFw0xNzA3MjQxNjA0 +MDFaGA8yMTE3MDYzMDE2MDQwMVowWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNv +bWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAG +A1UEAxMJdGVzdENlcnQyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +0brwpJYN2ytPWzRBtZSVc3dhkQlA59AzxzqeLLkano0Pxo9NIc3T/y58nnRI8uaS +I1P7BzUfJTiUEvmAtX8NggqKK4ld/gPrU+IRww1CUYS4KCkA/0d0ctPy0JwBCjD+ +b57G3rmNE8c+0jns6J96ZzNtqmv6N+ZlFBAXm1p4S+k0kGi5+hoQ6H7SYXjk2lG+ +r/8jPQEjy/NSdw1dcCA0Nc6o+hPr32927dS6J9KOhBeXNYUNdbuDDmroM9/gN2e/ +YMSA1olLeDPQ7Xvhk0PIyEDnHh83AffPCx5yM3htVRGddjIsPAVUJEL3z5leJtxe +fzyPghOhHJY0PXqznDQTcwIDAQABo4G+MIG7MB0GA1UdDgQWBBRP0IJqv/5rQ4Uf +SByl77dJeEapRDCBiwYDVR0jBIGDMIGAgBRP0IJqv/5rQ4UfSByl77dJeEapRKFd +pFswWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoT +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAxMJdGVzdENlcnQyggkA +3kxdkOJzupgwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEA0RIMHc10 +wHHPMh9UflqBgDMF7gfbOL0juJfGloAOcohWWfMZBBJ0CQKMy3xRyoK3HmbW1eeb +iATjesw7t4VEAwf7mgKAd+eTfWYB952uq5qYJ2TI28mSofEq1Wz3RmrNkC1KCBs1 +u+YMFGwyl6necV9zKCeiju4jeovI1GA38TvH7MgYln6vMJ+FbgOXj7XCpek7dQiY +KGaeSSH218mGNQaWRQw2Sm3W6cFdANoCJUph4w18s7gjtFpfV63s80hXRps+vEyv +jEQMEQpG8Ss7HGJLGLBw/xAmG0e//XS/o2dDonbGbvzToFByz8OGxjMhk6yV6hdd ++iyvsLAw/MYMSA== +-----END CERTIFICATE----- +` + +// testCert is a small helper to get a test x509.Certificate from the PEM constants +func testCert(t *testing.T, pemString string) *x509.Certificate { + // Decode the example certificate from a PEM file into a PEM block + pemBlock, _ := pem.Decode([]byte(pemString)) + if pemBlock == nil { + t.Fatal("failed to parse test certificate PEM") + return nil + } + + // Parse the PEM block into an x509.Certificate + result, err := x509.ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("failed to parse test certificate: %v", err) + return nil + } + return result +} + +func TestSet(t *testing.T) { + s := NewSet() + if !s.Empty() { + t.Error("expected a new set to be empty") + return + } + err := s.Allow("xyz") + if err == nil || !s.Empty() { + t.Error("expected allowing junk to fail") + return + } + + err = s.Allow("0011223344") + if err == nil || !s.Empty() { + t.Error("expected allowing something too short to fail") + return + } + + err = s.Allow(expectedHash + expectedHash) + if err == nil || !s.Empty() { + t.Error("expected allowing something too long to fail") + return + } + + err = s.CheckAny([]*x509.Certificate{testCert(t, testCertPEM)}) + if err == nil { + t.Error("expected test cert to not be allowed (yet)") + return + } + + err = s.Allow(strings.ToUpper(expectedHash)) + if err != nil || s.Empty() { + t.Error("expected allowing uppercase expectedHash to succeed") + return + } + + err = s.CheckAny([]*x509.Certificate{testCert(t, testCertPEM)}) + if err != nil { + t.Errorf("expected test cert to be allowed, but got back: %v", err) + return + } + + err = s.CheckAny([]*x509.Certificate{testCert(t, testCert2PEM)}) + if err == nil { + t.Error("expected the second test cert to be disallowed") + return + } + + s = NewSet() // keep set empty + hashes := []string{ + `sha256:0000000000000000000000000000000000000000000000000000000000000000`, + `sha256:0000000000000000000000000000000000000000000000000000000000000001`, + } + err = s.Allow(hashes...) + if err != nil || len(s.sha256Hashes) != 2 { + t.Error("expected allowing multiple hashes to succeed") + return + } +} + +func TestHash(t *testing.T) { + actualHash := Hash(testCert(t, testCertPEM)) + if actualHash != expectedHash { + t.Errorf( + "failed to Hash() to the expected value\n\texpected: %q\n\t actual: %q", + expectedHash, + actualHash, + ) + } +} diff --git a/pkg/util/secret.go b/pkg/util/secret.go index 7e0ce0920..a8aa09290 100644 --- a/pkg/util/secret.go +++ b/pkg/util/secret.go @@ -83,3 +83,17 @@ func PatchSecret(client kubeclient.Interface, namespace, name string, pt types.P } return nil } + +// CreateOrUpdateSecret creates a Secret if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateOrUpdateSecret(client kubeclient.Interface, secret *corev1.Secret) error { + if _, err := client.CoreV1().Secrets(secret.ObjectMeta.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create secret, err: %w", err) + } + + if _, err := client.CoreV1().Secrets(secret.ObjectMeta.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("unable to update secret, err: %w", err) + } + } + return nil +} diff --git a/vendor/k8s.io/cluster-bootstrap/util/secrets/secrets.go b/vendor/k8s.io/cluster-bootstrap/util/secrets/secrets.go new file mode 100644 index 000000000..73894ee0d --- /dev/null +++ b/vendor/k8s.io/cluster-bootstrap/util/secrets/secrets.go @@ -0,0 +1,112 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secrets + +import ( + "regexp" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/cluster-bootstrap/token/api" + legacyutil "k8s.io/cluster-bootstrap/token/util" + "k8s.io/klog/v2" +) + +var ( + secretNameRe = regexp.MustCompile(`^` + regexp.QuoteMeta(api.BootstrapTokenSecretPrefix) + `([a-z0-9]{6})$`) +) + +// GetData returns the string value for the given key in the specified Secret +// If there is an error or if the key doesn't exist, an empty string is returned. +func GetData(secret *v1.Secret, key string) string { + if secret.Data == nil { + return "" + } + if val, ok := secret.Data[key]; ok { + return string(val) + } + return "" +} + +// HasExpired will identify whether the secret expires +func HasExpired(secret *v1.Secret, currentTime time.Time) bool { + _, expired := GetExpiration(secret, currentTime) + + return expired +} + +// GetExpiration checks if the secret expires +// isExpired indicates if the secret is already expired. +// timeRemaining indicates how long until it does expire. +// if the secret has no expiration timestamp, returns 0, false. +// if there is an error parsing the secret's expiration timestamp, returns 0, true. +func GetExpiration(secret *v1.Secret, currentTime time.Time) (timeRemaining time.Duration, isExpired bool) { + expiration := GetData(secret, api.BootstrapTokenExpirationKey) + if len(expiration) == 0 { + return 0, false + } + expTime, err := time.Parse(time.RFC3339, expiration) + if err != nil { + klog.V(3).Infof("Unparseable expiration time (%s) in %s/%s Secret: %v. Treating as expired.", + expiration, secret.Namespace, secret.Name, err) + return 0, true + } + + timeRemaining = expTime.Sub(currentTime) + if timeRemaining <= 0 { + klog.V(3).Infof("Expired bootstrap token in %s/%s Secret: %v", + secret.Namespace, secret.Name, expiration) + return 0, true + } + return timeRemaining, false +} + +// ParseName parses the name of the secret to extract the secret ID. +func ParseName(name string) (secretID string, ok bool) { + r := secretNameRe.FindStringSubmatch(name) + if r == nil { + return "", false + } + return r[1], true +} + +// GetGroups loads and validates the bootstrapapi.BootstrapTokenExtraGroupsKey +// key from the bootstrap token secret, returning a list of group names or an +// error if any of the group names are invalid. +func GetGroups(secret *v1.Secret) ([]string, error) { + // always include the default group + groups := sets.NewString(api.BootstrapDefaultGroup) + + // grab any extra groups and if there are none, return just the default + extraGroupsString := GetData(secret, api.BootstrapTokenExtraGroupsKey) + if extraGroupsString == "" { + return groups.List(), nil + } + + // validate the names of the extra groups + for _, group := range strings.Split(extraGroupsString, ",") { + if err := legacyutil.ValidateBootstrapGroupName(group); err != nil { + return nil, err + } + groups.Insert(group) + } + + // return the result as a deduplicated, sorted list + return groups.List(), nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index be583c0ba..3716c60a4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1299,6 +1299,7 @@ k8s.io/client-go/util/workqueue ## explicit; go 1.16 k8s.io/cluster-bootstrap/token/api k8s.io/cluster-bootstrap/token/util +k8s.io/cluster-bootstrap/util/secrets # k8s.io/code-generator v0.24.2 ## explicit; go 1.16 k8s.io/code-generator