diff --git a/deploy/example/storageclass-smb-krb5.yaml b/deploy/example/storageclass-smb-krb5.yaml new file mode 100644 index 000000000..4ab4d3e30 --- /dev/null +++ b/deploy/example/storageclass-smb-krb5.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: smb-krb5 +provisioner: smb.csi.k8s.io +parameters: + # On Windows, "*.default.svc.cluster.local" could not be recognized by csi-proxy + source: "//smb-server.default.svc.cluster.local/share" + # if csi.storage.k8s.io/provisioner-secret is provided, will create a sub directory + # with PV name under source + csi.storage.k8s.io/provisioner-secret-name: "smbcreds-krb5" + csi.storage.k8s.io/provisioner-secret-namespace: "default" + csi.storage.k8s.io/node-stage-secret-name: "smbcreds-krb5" + csi.storage.k8s.io/node-stage-secret-namespace: "default" +volumeBindingMode: Immediate +mountOptions: + - sec=krb5 + - cruid=1000 + - seal + - vers=3.0 + - nosuid + - noexec + - dir_mode=0777 + - file_mode=0777 + - uid=1001 + - gid=1001 + - noperm + - mfsymlinks + - cache=strict + - noserverino # required to prevent data corruption diff --git a/docs/driver-parameters.md b/docs/driver-parameters.md index ad8cb0113..37c90b10e 100644 --- a/docs/driver-parameters.md +++ b/docs/driver-parameters.md @@ -34,6 +34,45 @@ nodeStageSecretRef.namespace | namespace where the secret is | k8s namespace | kubectl create secret generic smbcreds --from-literal username=USERNAME --from-literal password="PASSWORD" ``` +### Kerberos ticket support for Linux + + + + +#### These are the conditions that must be met: + - Kerberos support should be set up and cifs-utils must be installed on every node. + - The directory /var/lib/kubelet/kerberos/ needs to exist, and it will hold kerberos credential cache files for various users. + - This directory is shared between the host and the smb container. + - The admin is responsible for cleaning up the directory on each node as they deem appropriate. It's important to note that unmounting doesn't delete the cache file. + - Each node should know to look up in that directory, here's example script for that, expected to be run on node provision: +```console +mkdir -p /etc/krb5.conf.d/ +echo "[libdefaults] +default_ccache_name = FILE:/var/lib/kubelet/kerberos/krb5cc_%{uid}" > /etc/krb5.conf.d/ccache.conf + ``` + - Mount flags should include **sec=krb5,cruid=1000** + - sec=krb5 enables using credential cache + - cruid=1000 provides information for what user credential cache will be looked up. This should match the secret entry. + +#### Pass kerberos ticket in kubernetes secret +To pass a ticket through secret, it needs to be acquired. Here's example how it can be done: + +```console +export KRB5CCNAME=/tmp/ccache # Use temporary file for the cache +kinit USERNAME # Log in into domain +kvno cifs/lowercase_server_name # Acquire ticket for the needed share, it'll be written to the cache file +CCACHE=$(base64 -w 0 $KRB5CCNAME) # Get Base64-encoded cache +``` + +And passing the actual ticket to the secret, instead of the password. +Note that key for the ticket has included credential id, that must match exactly `cruid=` mount flag. +In theory, nothing prevents from having more than single ticket cache in the same secret. +```console +kubectl create secret generic smbcreds-krb5 --from-literal krb5cc_1000=$CCACHE +``` + +> See example of the [StorageClass](../deploy/example/storageclass-smb-krb5.yaml) + ### Tips #### `subDir` parameter supports following pv/pvc metadata conversion > if `subDir` value contains following string, it would be converted into corresponding pv/pvc name or namespace diff --git a/pkg/smb/nodeserver.go b/pkg/smb/nodeserver.go index e5726dec1..cf8cf8cf8 100644 --- a/pkg/smb/nodeserver.go +++ b/pkg/smb/nodeserver.go @@ -17,10 +17,12 @@ limitations under the License. package smb import ( + "encoding/base64" "fmt" "os" "path/filepath" "runtime" + "strconv" "strings" "time" @@ -182,10 +184,14 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe sensitiveMountOptions = []string{password} } } else { + var useKerberosCache, err = ensureKerberosCache(mountFlags, secrets) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("Error writing kerberos cache: %v", err)) + } if err := os.MkdirAll(targetPath, 0750); err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err)) } - if requireUsernamePwdOption { + if requireUsernamePwdOption && !useKerberosCache { sensitiveMountOptions = []string{fmt.Sprintf("%s=%s,%s=%s", usernameField, username, passwordField, password)} } mountOptions = mountFlags @@ -422,3 +428,88 @@ func checkGidPresentInMountFlags(mountFlags []string) bool { } return false } + +func hasKerberosMountOption(mountFlags []string) bool { + for _, mountFlag := range mountFlags { + if strings.HasPrefix(mountFlag, "sec=krb5") { + return true + } + } + return false +} + +func getCredUID(mountFlags []string) (int, error) { + var cruidPrefix = "cruid=" + for _, mountFlag := range mountFlags { + if strings.HasPrefix(mountFlag, cruidPrefix) { + return strconv.Atoi(strings.TrimPrefix(mountFlag, cruidPrefix)) + } + } + return -1, fmt.Errorf("Can't find credUid in mount flags") +} + +func getKrb5CcacheName(credUID int) string { + return fmt.Sprintf("%s%d", krb5Prefix, credUID) +} + +func getKrb5CacheFileName(credUID int) string { + return fmt.Sprintf("%s%s%d", krb5CacheDirectory, krb5Prefix, credUID) +} +func kerberosCacheDirectoryExists() (bool, error) { + _, err := os.Stat(krb5CacheDirectory) + if os.IsNotExist(err) { + return false, status.Error(codes.Internal, fmt.Sprintf("Directory for kerberos caches must exist, it will not be created: %s: %v", krb5CacheDirectory, err)) + } else if err != nil { + return false, err + } + return true, nil +} + +func getKerberosCache(credUID int, secrets map[string]string) (string, []byte, error) { + var krb5CcacheName = getKrb5CcacheName(credUID) + var krb5CcacheContent string + for k, v := range secrets { + switch strings.ToLower(k) { + case krb5CcacheName: + krb5CcacheContent = v + } + } + if krb5CcacheContent == "" { + return "", nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Empty kerberos cache in key %s", krb5CcacheName)) + } + content, err := base64.StdEncoding.DecodeString(krb5CcacheContent) + if err != nil { + return "", nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Malformed kerberos cache in key %s, expected to be in base64 form: %v", krb5CcacheName, err)) + } + var krb5CacheFileName = getKrb5CacheFileName(credUID) + + return krb5CacheFileName, content, nil +} + +func ensureKerberosCache(mountFlags []string, secrets map[string]string) (bool, error) { + var securityIsKerberos = hasKerberosMountOption(mountFlags) + if securityIsKerberos { + _, err := kerberosCacheDirectoryExists() + if err != nil { + return false, err + } + credUID, err := getCredUID(mountFlags) + if err != nil { + return false, err + } + krb5CacheFileName, content, err := getKerberosCache(credUID, secrets) + if err != nil { + return false, err + } + err = os.WriteFile(krb5CacheFileName, content, os.FileMode(0700)) + if err != nil { + return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't write kerberos cache to file %s: %v", krb5CacheFileName, err)) + } + err = os.Chown(krb5CacheFileName, credUID, credUID) + if err != nil { + return false, status.Error(codes.Internal, fmt.Sprintf("Couldn't chown kerberos cache %s to user %d: %v", krb5CacheFileName, credUID, err)) + } + return true, nil + } + return false, nil +} diff --git a/pkg/smb/nodeserver_test.go b/pkg/smb/nodeserver_test.go index db4e22572..e392346ea 100644 --- a/pkg/smb/nodeserver_test.go +++ b/pkg/smb/nodeserver_test.go @@ -18,12 +18,14 @@ package smb import ( "context" + "encoding/base64" "errors" "fmt" "os" "path/filepath" "reflect" "runtime" + "strconv" "strings" "syscall" "testing" @@ -690,6 +692,161 @@ func TestCheckGidPresentInMountFlags(t *testing.T) { } } +func TestHasKerberosMountOption(t *testing.T) { + tests := []struct { + desc string + MountFlags []string + result bool + }{ + { + desc: "[Success] Sec kerberos present in mount flags", + MountFlags: []string{"sec=krb5"}, + result: true, + }, + { + desc: "[Success] Sec kerberos present in mount flags", + MountFlags: []string{"sec=krb5i"}, + result: true, + }, + { + desc: "[Success] Sec kerberos not present in mount flags", + MountFlags: []string{}, + result: false, + }, + { + desc: "[Success] Sec kerberos not present in mount flags", + MountFlags: []string{"sec=ntlm"}, + result: false, + }, + } + + for _, test := range tests { + securityIsKerberos := hasKerberosMountOption(test.MountFlags) + if securityIsKerberos != test.result { + t.Errorf("[%s]: Expected result : %t, Actual result: %t", test.desc, test.result, securityIsKerberos) + } + } +} + +func TestGetCredUID(t *testing.T) { + _, convertErr := strconv.Atoi("foo") + tests := []struct { + desc string + MountFlags []string + result int + expectedErr error + }{ + { + desc: "[Success] Got correct credUID", + MountFlags: []string{"cruid=1000"}, + result: 1000, + expectedErr: nil, + }, + { + desc: "[Success] Got correct credUID", + MountFlags: []string{"cruid=0"}, + result: 0, + expectedErr: nil, + }, + { + desc: "[Error] Got error when no CredUID", + MountFlags: []string{}, + result: -1, + expectedErr: fmt.Errorf("Can't find credUid in mount flags"), + }, + { + desc: "[Error] Got error when CredUID is not an int", + MountFlags: []string{"cruid=foo"}, + result: 0, + expectedErr: convertErr, + }, + } + + for _, test := range tests { + credUID, err := getCredUID(test.MountFlags) + if credUID != test.result { + t.Errorf("[%s]: Expected result : %d, Actual result: %d", test.desc, test.result, credUID) + } + if !reflect.DeepEqual(err, test.expectedErr) { + t.Errorf("[%s]: Expected error : %v, Actual error: %v", test.desc, test.expectedErr, err) + } + } +} + +func TestGetKerberosCache(t *testing.T) { + ticket := []byte{'G', 'O', 'L', 'A', 'N', 'G'} + base64Ticket := base64.StdEncoding.EncodeToString(ticket) + credUID := 1000 + goodFileName := fmt.Sprintf("%s%s%d", krb5CacheDirectory, krb5Prefix, credUID) + krb5CcacheName := "krb5cc_1000" + + _, base64DecError := base64.StdEncoding.DecodeString("123") + tests := []struct { + desc string + credUID int + secrets map[string]string + expectedFileName string + expectedContent []byte + expectedErr error + }{ + { + desc: "[Success] Got correct filename and content", + credUID: 1000, + secrets: map[string]string{ + krb5CcacheName: base64Ticket, + }, + expectedFileName: goodFileName, + expectedContent: ticket, + expectedErr: nil, + }, + { + desc: "[Error] Throw error if credUID mismatch", + credUID: 1001, + secrets: map[string]string{ + krb5CcacheName: base64Ticket, + }, + expectedFileName: "", + expectedContent: nil, + expectedErr: status.Error(codes.InvalidArgument, fmt.Sprintf("Empty kerberos cache in key %s", "krb5cc_1001")), + }, + { + desc: "[Error] Throw error if ticket is empty in secret", + credUID: 1000, + secrets: map[string]string{ + krb5CcacheName: "", + }, + expectedFileName: "", + expectedContent: nil, + expectedErr: status.Error(codes.InvalidArgument, fmt.Sprintf("Empty kerberos cache in key %s", krb5CcacheName)), + }, + { + desc: "[Error] Throw error if ticket is invalid base64", + credUID: 1000, + secrets: map[string]string{ + krb5CcacheName: "123", + }, + expectedFileName: "", + expectedContent: nil, + expectedErr: status.Error(codes.InvalidArgument, fmt.Sprintf("Malformed kerberos cache in key %s, expected to be in base64 form: %v", krb5CcacheName, base64DecError)), + }, + } + + for _, test := range tests { + fileName, content, err := getKerberosCache(test.credUID, test.secrets) + if !reflect.DeepEqual(err, test.expectedErr) { + t.Errorf("[%s]: Expected error : %v, Actual error: %v", test.desc, test.expectedErr, err) + } else { + if fileName != test.expectedFileName { + t.Errorf("[%s]: Expected filename : %s, Actual result: %s", test.desc, test.expectedFileName, fileName) + } + if !reflect.DeepEqual(content, test.expectedContent) { + t.Errorf("[%s]: Expected content : %s, Actual content: %s", test.desc, test.expectedContent, content) + } + } + } + +} + func TestNodePublishVolumeIdempotentMount(t *testing.T) { if runtime.GOOS == "windows" || os.Getuid() != 0 { return diff --git a/pkg/smb/smb.go b/pkg/smb/smb.go index f25101df4..0f291f256 100644 --- a/pkg/smb/smb.go +++ b/pkg/smb/smb.go @@ -35,6 +35,8 @@ const ( sourceField = "source" subDirField = "subdir" domainField = "domain" + krb5Prefix = "krb5cc_" + krb5CacheDirectory = "/var/lib/kubelet/kerberos/" mountOptionsField = "mountoptions" defaultDomainName = "AZURE" pvcNameKey = "csi.storage.k8s.io/pvc/name"