Cleaning kerberos cache files on unstage phase

Extending initial support for kerberos-based ticket mounting.
Previously, cache has been written directly to the expected file - krb5cc_*.
With this change, cache is written to the base64(volumeID) file,
and then symlink krb5cc_* created pointing to the file with the cache.
During unstage phase only volumeID and mount path is available, and
having volumeID it's possible to do the cleanup with simple traversing
directory containing the caches.

docs/driver-parameters.md

@@ -43,16 +43,17 @@ kubectl create secret generic smbcreds --from-literal username=USERNAME --from-l
  - Kerberos support should be set up and cifs-utils must be installed on every node.
  - The directory /var/lib/kubelet/kerberos/ needs to exist, and it will hold kerberos credential cache files for various users.
  - This directory is shared between the host and the smb container.
- - The admin is responsible for cleaning up the directory on each node as they deem appropriate. It's important to note that unmounting doesn't delete the cache file.
+ - The kerberos cache files are created for each volume and cleaned up during UnstageVolume phase
  - Each node should know to look up in that directory, here's example script for that, expected to be run on node provision:
 ```console
 mkdir -p /etc/krb5.conf.d/
 echo "[libdefaults]
 default_ccache_name = FILE:/var/lib/kubelet/kerberos/krb5cc_%{uid}" > /etc/krb5.conf.d/ccache.conf
    ```
- - Mount flags should include **sec=krb5,cruid=1000**
+ - Mount flags should include **sec=krb5,uid=1000,cruid=1000**
    - sec=krb5 enables using credential cache
    - cruid=1000 provides information for what user credential cache will be looked up. This should match the secret entry.
+   - uid=1000 is the owner of mounted files. This doesn't have to be the same as cruid.
 
 #### Pass kerberos ticket in kubernetes secret 
 To pass a ticket through secret, it needs to be acquired. Here's example how it can be done:

pkg/smb/nodeserver.go

@@ -184,7 +184,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 			sensitiveMountOptions = []string{password}
 		}
 	} else {
-		var useKerberosCache, err = ensureKerberosCache(mountFlags, secrets)
+		var useKerberosCache, err = ensureKerberosCache(volumeID, mountFlags, secrets)
 		if err != nil {
 			return nil, status.Error(codes.Internal, fmt.Sprintf("Error writing kerberos cache: %v", err))
 		}
@@ -262,6 +262,10 @@ func (d *Driver) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolu
 		return nil, status.Errorf(codes.Internal, "failed to unmount staging target %q: %v", stagingTargetPath, err)
 	}
 
+	if err := deleteKerberosCache(volumeID); err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to delete kerberos cache: %v", err)
+	}
+
 	klog.V(2).Infof("NodeUnstageVolume: unmount volume %s on %s successfully", volumeID, stagingTargetPath)
 	return &csi.NodeUnstageVolumeResponse{}, nil
 }
@@ -452,9 +456,16 @@ func getKrb5CcacheName(credUID int) string {
 	return fmt.Sprintf("%s%d", krb5Prefix, credUID)
 }
 
-func getKrb5CacheFileName(credUID int) string {
-	return fmt.Sprintf("%s%s%d", krb5CacheDirectory, krb5Prefix, credUID)
+// returns absolute path for name of file inside krb5CacheDirectory
+func getKerberosFilePath(fileName string) string {
+	return fmt.Sprintf("%s%s", krb5CacheDirectory, fileName)
 }
+
+func volumeKerberosCacheName(volumeID string) string {
+	encoded := base64.StdEncoding.EncodeToString([]byte(volumeID))
+	return strings.ReplaceAll(strings.ReplaceAll(encoded, "/", "-"), "+", "_")
+}
+
 func kerberosCacheDirectoryExists() (bool, error) {
 	_, err := os.Stat(krb5CacheDirectory)
 	if os.IsNotExist(err) {
@@ -481,12 +492,15 @@ func getKerberosCache(credUID int, secrets map[string]string) (string, []byte, e
 	if err != nil {
 		return "", nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Malformed kerberos cache in key %s, expected to be in base64 form: %v", krb5CcacheName, err))
 	}
-	var krb5CacheFileName = getKrb5CacheFileName(credUID)
+	var krb5CacheFileName = getKerberosFilePath(getKrb5CcacheName(credUID))
 
 	return krb5CacheFileName, content, nil
 }
 
-func ensureKerberosCache(mountFlags []string, secrets map[string]string) (bool, error) {
+// Create kerberos cache in the file based on the VolumeID, so it can be cleaned up during unstage
+// At the same time, kerberos expects to find cache in file named "krb5cc_*", so creating symlink
+// will allow both clean up and serving proper cache to the kerberos.
+func ensureKerberosCache(volumeID string, mountFlags []string, secrets map[string]string) (bool, error) {
 	var securityIsKerberos = hasKerberosMountOption(mountFlags)
 	if securityIsKerberos {
 		_, err := kerberosCacheDirectoryExists()
@@ -501,15 +515,77 @@ func ensureKerberosCache(mountFlags []string, secrets map[string]string) (bool,
 		if err != nil {
 			return false, err
 		}
-		err = os.WriteFile(krb5CacheFileName, content, os.FileMode(0700))
-		if err != nil {
-			return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't write kerberos cache to file %s: %v", krb5CacheFileName, err))
+		// Write cache into volumeId-based filename, so it can be cleaned up later
+		volumeIDCacheFileName := volumeKerberosCacheName(volumeID)
+
+		volumeIDCacheAbsolutePath := getKerberosFilePath(volumeIDCacheFileName)
+		if err := os.WriteFile(volumeIDCacheAbsolutePath, content, os.FileMode(0700)); err != nil {
+			return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't write kerberos cache to file %s: %v", volumeIDCacheAbsolutePath, err))
 		}
-		err = os.Chown(krb5CacheFileName, credUID, credUID)
-		if err != nil {
-			return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't chown kerberos cache %s to user %d: %v", krb5CacheFileName, credUID, err))
+		if err := os.Chown(volumeIDCacheAbsolutePath, credUID, credUID); err != nil {
+			return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't chown kerberos cache %s to user %d: %v", volumeIDCacheAbsolutePath, credUID, err))
 		}
+
+		if _, err := os.Stat(krb5CacheFileName); os.IsNotExist(err) {
+			klog.Warningf("symlink file doesn't exist, it'll be created [%s]", krb5CacheFileName)
+		} else {
+			if err := os.Remove(krb5CacheFileName); err != nil {
+				klog.Warningf("couldn't delete the file [%s]", krb5CacheFileName)
+			}
+		}
+
+		// Create symlink to the cache file with expected name
+		if err := os.Symlink(volumeIDCacheAbsolutePath, krb5CacheFileName); err != nil {
+			return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't create symlink to a cache file %s->%s to user %d: %v", krb5CacheFileName, volumeIDCacheFileName, credUID, err))
+		}
+
 		return true, nil
 	}
 	return false, nil
 }
+
+func deleteKerberosCache(volumeID string) error {
+	exists, err := kerberosCacheDirectoryExists()
+	// If not supported, simply return
+	if !exists {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	volumeIDCacheFileName := volumeKerberosCacheName(volumeID)
+
+	var volumeIDCacheAbsolutePath = getKerberosFilePath(volumeIDCacheFileName)
+	_, err = os.Stat(volumeIDCacheAbsolutePath)
+	// Not created or already removed
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	// If file with cache exists, full clean means removing symlinks to the file.
+	dirEntries, _ := os.ReadDir(krb5CacheDirectory)
+	for _, dirEntry := range dirEntries {
+		filePath := getKerberosFilePath(dirEntry.Name())
+		lStat, _ := os.Lstat(filePath)
+		// If it's a symlink, checking if it's pointing to the volume file in question
+		if lStat != nil {
+			target, _ := os.Readlink(filePath)
+			if target == volumeIDCacheAbsolutePath {
+				err = os.Remove(filePath)
+				if err != nil {
+					klog.Errorf("Error removing symlink to kerberos ticket cache: %s (%v)", filePath, err)
+				}
+			}
+		}
+	}
+
+	err = os.Remove(volumeIDCacheAbsolutePath)
+	if err != nil {
+		klog.Errorf("Error removing symlink to kerberos ticket cache: %s (%v)", volumeIDCacheAbsolutePath, err)
+	}
+
+	return nil
+}

pkg/smb/nodeserver_test.go

@@ -692,6 +692,29 @@ func TestCheckGidPresentInMountFlags(t *testing.T) {
 	}
 }
 
+func TestVolumeKerberosCacheName(t *testing.T) {
+	tests := []struct {
+		name string
+	}{
+		{
+			name: "s", // short name
+		},
+		{
+			name: "Volume Handle##unique suffix",
+		},
+		{
+			name: "Volume With Spaces and Slashes // and symbols that produce /+ after base64 ???????~~~~~~~~",
+		},
+	}
+
+	for _, test := range tests {
+		fileName := volumeKerberosCacheName(test.name)
+		if strings.Contains(fileName, "/") || strings.Contains(fileName, "+") {
+			t.Errorf("[%s]: Expected result should not contain / or +, Actual result: %s", test.name, fileName)
+		}
+	}
+}
+
 func TestHasKerberosMountOption(t *testing.T) {
 	tests := []struct {
 		desc       string