diff --git a/docs/driver-parameters.md b/docs/driver-parameters.md index 37c90b10e..d4ea7febd 100644 --- a/docs/driver-parameters.md +++ b/docs/driver-parameters.md @@ -43,16 +43,17 @@ kubectl create secret generic smbcreds --from-literal username=USERNAME --from-l - Kerberos support should be set up and cifs-utils must be installed on every node. - The directory /var/lib/kubelet/kerberos/ needs to exist, and it will hold kerberos credential cache files for various users. - This directory is shared between the host and the smb container. - - The admin is responsible for cleaning up the directory on each node as they deem appropriate. It's important to note that unmounting doesn't delete the cache file. + - The kerberos cache files are created for each volume and cleaned up during UnstageVolume phase - Each node should know to look up in that directory, here's example script for that, expected to be run on node provision: ```console mkdir -p /etc/krb5.conf.d/ echo "[libdefaults] default_ccache_name = FILE:/var/lib/kubelet/kerberos/krb5cc_%{uid}" > /etc/krb5.conf.d/ccache.conf ``` - - Mount flags should include **sec=krb5,cruid=1000** + - Mount flags should include **sec=krb5,uid=1000,cruid=1000** - sec=krb5 enables using credential cache - cruid=1000 provides information for what user credential cache will be looked up. This should match the secret entry. + - uid=1000 is the owner of mounted files. This doesn't have to be the same as cruid. #### Pass kerberos ticket in kubernetes secret To pass a ticket through secret, it needs to be acquired. Here's example how it can be done: diff --git a/pkg/smb/nodeserver.go b/pkg/smb/nodeserver.go index cf8cf8cf8..59b63f91a 100644 --- a/pkg/smb/nodeserver.go +++ b/pkg/smb/nodeserver.go @@ -184,7 +184,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe sensitiveMountOptions = []string{password} } } else { - var useKerberosCache, err = ensureKerberosCache(mountFlags, secrets) + var useKerberosCache, err = ensureKerberosCache(volumeID, mountFlags, secrets) if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("Error writing kerberos cache: %v", err)) } @@ -262,6 +262,10 @@ func (d *Driver) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolu return nil, status.Errorf(codes.Internal, "failed to unmount staging target %q: %v", stagingTargetPath, err) } + if err := deleteKerberosCache(volumeID); err != nil { + return nil, status.Errorf(codes.Internal, "failed to delete kerberos cache: %v", err) + } + klog.V(2).Infof("NodeUnstageVolume: unmount volume %s on %s successfully", volumeID, stagingTargetPath) return &csi.NodeUnstageVolumeResponse{}, nil } @@ -452,9 +456,16 @@ func getKrb5CcacheName(credUID int) string { return fmt.Sprintf("%s%d", krb5Prefix, credUID) } -func getKrb5CacheFileName(credUID int) string { - return fmt.Sprintf("%s%s%d", krb5CacheDirectory, krb5Prefix, credUID) +// returns absolute path for name of file inside krb5CacheDirectory +func getKerberosFilePath(fileName string) string { + return fmt.Sprintf("%s%s", krb5CacheDirectory, fileName) } + +func volumeKerberosCacheName(volumeID string) string { + encoded := base64.StdEncoding.EncodeToString([]byte(volumeID)) + return strings.ReplaceAll(strings.ReplaceAll(encoded, "/", "-"), "+", "_") +} + func kerberosCacheDirectoryExists() (bool, error) { _, err := os.Stat(krb5CacheDirectory) if os.IsNotExist(err) { @@ -481,12 +492,15 @@ func getKerberosCache(credUID int, secrets map[string]string) (string, []byte, e if err != nil { return "", nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Malformed kerberos cache in key %s, expected to be in base64 form: %v", krb5CcacheName, err)) } - var krb5CacheFileName = getKrb5CacheFileName(credUID) + var krb5CacheFileName = getKerberosFilePath(getKrb5CcacheName(credUID)) return krb5CacheFileName, content, nil } -func ensureKerberosCache(mountFlags []string, secrets map[string]string) (bool, error) { +// Create kerberos cache in the file based on the VolumeID, so it can be cleaned up during unstage +// At the same time, kerberos expects to find cache in file named "krb5cc_*", so creating symlink +// will allow both clean up and serving proper cache to the kerberos. +func ensureKerberosCache(volumeID string, mountFlags []string, secrets map[string]string) (bool, error) { var securityIsKerberos = hasKerberosMountOption(mountFlags) if securityIsKerberos { _, err := kerberosCacheDirectoryExists() @@ -501,15 +515,77 @@ func ensureKerberosCache(mountFlags []string, secrets map[string]string) (bool, if err != nil { return false, err } - err = os.WriteFile(krb5CacheFileName, content, os.FileMode(0700)) - if err != nil { - return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't write kerberos cache to file %s: %v", krb5CacheFileName, err)) + // Write cache into volumeId-based filename, so it can be cleaned up later + volumeIDCacheFileName := volumeKerberosCacheName(volumeID) + + volumeIDCacheAbsolutePath := getKerberosFilePath(volumeIDCacheFileName) + if err := os.WriteFile(volumeIDCacheAbsolutePath, content, os.FileMode(0700)); err != nil { + return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't write kerberos cache to file %s: %v", volumeIDCacheAbsolutePath, err)) } - err = os.Chown(krb5CacheFileName, credUID, credUID) - if err != nil { - return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't chown kerberos cache %s to user %d: %v", krb5CacheFileName, credUID, err)) + if err := os.Chown(volumeIDCacheAbsolutePath, credUID, credUID); err != nil { + return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't chown kerberos cache %s to user %d: %v", volumeIDCacheAbsolutePath, credUID, err)) } + + if _, err := os.Stat(krb5CacheFileName); os.IsNotExist(err) { + klog.Warningf("symlink file doesn't exist, it'll be created [%s]", krb5CacheFileName) + } else { + if err := os.Remove(krb5CacheFileName); err != nil { + klog.Warningf("couldn't delete the file [%s]", krb5CacheFileName) + } + } + + // Create symlink to the cache file with expected name + if err := os.Symlink(volumeIDCacheAbsolutePath, krb5CacheFileName); err != nil { + return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't create symlink to a cache file %s->%s to user %d: %v", krb5CacheFileName, volumeIDCacheFileName, credUID, err)) + } + return true, nil } return false, nil } + +func deleteKerberosCache(volumeID string) error { + exists, err := kerberosCacheDirectoryExists() + // If not supported, simply return + if !exists { + return nil + } + if err != nil { + return err + } + + volumeIDCacheFileName := volumeKerberosCacheName(volumeID) + + var volumeIDCacheAbsolutePath = getKerberosFilePath(volumeIDCacheFileName) + _, err = os.Stat(volumeIDCacheAbsolutePath) + // Not created or already removed + if os.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + + // If file with cache exists, full clean means removing symlinks to the file. + dirEntries, _ := os.ReadDir(krb5CacheDirectory) + for _, dirEntry := range dirEntries { + filePath := getKerberosFilePath(dirEntry.Name()) + lStat, _ := os.Lstat(filePath) + // If it's a symlink, checking if it's pointing to the volume file in question + if lStat != nil { + target, _ := os.Readlink(filePath) + if target == volumeIDCacheAbsolutePath { + err = os.Remove(filePath) + if err != nil { + klog.Errorf("Error removing symlink to kerberos ticket cache: %s (%v)", filePath, err) + } + } + } + } + + err = os.Remove(volumeIDCacheAbsolutePath) + if err != nil { + klog.Errorf("Error removing symlink to kerberos ticket cache: %s (%v)", volumeIDCacheAbsolutePath, err) + } + + return nil +} diff --git a/pkg/smb/nodeserver_test.go b/pkg/smb/nodeserver_test.go index e392346ea..fa8579866 100644 --- a/pkg/smb/nodeserver_test.go +++ b/pkg/smb/nodeserver_test.go @@ -692,6 +692,29 @@ func TestCheckGidPresentInMountFlags(t *testing.T) { } } +func TestVolumeKerberosCacheName(t *testing.T) { + tests := []struct { + name string + }{ + { + name: "s", // short name + }, + { + name: "Volume Handle##unique suffix", + }, + { + name: "Volume With Spaces and Slashes // and symbols that produce /+ after base64 ???????~~~~~~~~", + }, + } + + for _, test := range tests { + fileName := volumeKerberosCacheName(test.name) + if strings.Contains(fileName, "/") || strings.Contains(fileName, "+") { + t.Errorf("[%s]: Expected result should not contain / or +, Actual result: %s", test.name, fileName) + } + } +} + func TestHasKerberosMountOption(t *testing.T) { tests := []struct { desc string