From 995f3538dc89dc59a6f97d912bad9a19f6f328fe Mon Sep 17 00:00:00 2001 From: Dipti Pai Date: Tue, 26 Aug 2025 10:30:13 -0700 Subject: [PATCH] [RFC-0010] Add multi-tenant workload identity support for Azure Blob Storage Signed-off-by: Dipti Pai --- api/v1/bucket_types.go | 2 +- .../source.toolkit.fluxcd.io_buckets.yaml | 6 +- docs/spec/v1/buckets.md | 86 +++++-------------- internal/bucket/azure/blob.go | 39 +++------ internal/bucket/azure/blob_test.go | 5 +- internal/controller/bucket_controller.go | 3 +- 6 files changed, 44 insertions(+), 97 deletions(-) diff --git a/api/v1/bucket_types.go b/api/v1/bucket_types.go index 764ee1bb..3e68e502 100644 --- a/api/v1/bucket_types.go +++ b/api/v1/bucket_types.go @@ -52,7 +52,7 @@ const ( // +kubebuilder:validation:XValidation:rule="self.provider != 'generic' || !has(self.sts) || self.sts.provider == 'ldap'", message="'ldap' is the only supported STS provider for the 'generic' Bucket provider" // +kubebuilder:validation:XValidation:rule="!has(self.sts) || self.sts.provider != 'aws' || !has(self.sts.secretRef)", message="spec.sts.secretRef is not required for the 'aws' STS provider" // +kubebuilder:validation:XValidation:rule="!has(self.sts) || self.sts.provider != 'aws' || !has(self.sts.certSecretRef)", message="spec.sts.certSecretRef is not required for the 'aws' STS provider" -// +kubebuilder:validation:XValidation:rule="self.provider == 'gcp' || self.provider == 'aws' || !has(self.serviceAccountName)", message="ServiceAccountName is only supported for the 'gcp' and 'aws' Bucket providers" +// +kubebuilder:validation:XValidation:rule="self.provider != 'generic' || !has(self.serviceAccountName)", message="ServiceAccountName is not supported for the 'generic' Bucket provider" // +kubebuilder:validation:XValidation:rule="!has(self.secretRef) || !has(self.serviceAccountName)", message="cannot set both .spec.secretRef and .spec.serviceAccountName" type BucketSpec struct { // Provider of the object storage bucket. diff --git a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml index 445beaf5..6f89f666 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml @@ -239,9 +239,9 @@ spec: rule: '!has(self.sts) || self.sts.provider != ''aws'' || !has(self.sts.secretRef)' - message: spec.sts.certSecretRef is not required for the 'aws' STS provider rule: '!has(self.sts) || self.sts.provider != ''aws'' || !has(self.sts.certSecretRef)' - - message: ServiceAccountName is only supported for the 'gcp' and 'aws' - Bucket providers - rule: self.provider == 'gcp' || self.provider == 'aws' || !has(self.serviceAccountName) + - message: ServiceAccountName is not supported for the 'generic' Bucket + provider + rule: self.provider != 'generic' || !has(self.serviceAccountName) - message: cannot set both .spec.secretRef and .spec.serviceAccountName rule: '!has(self.secretRef) || !has(self.serviceAccountName)' status: diff --git a/docs/spec/v1/buckets.md b/docs/spec/v1/buckets.md index 03e65165..077ac952 100644 --- a/docs/spec/v1/buckets.md +++ b/docs/spec/v1/buckets.md @@ -567,83 +567,39 @@ metadata: spec: interval: 5m0s provider: azure - bucketName: testsas - endpoint: https://testfluxsas.blob.core.windows.net + bucketName: testwi + endpoint: https://testfluxwi.blob.core.windows.net ``` -##### Deprecated: Managed Identity with AAD Pod Identity +##### Azure Object-Level Workload Identity example -If you are using [aad pod identity](https://azure.github.io/aad-pod-identity/docs), -You need to create an Azure Identity and give it access to Azure Blob Storage. - -```sh -export IDENTITY_NAME="blob-access" - -az role assignment create --role "Storage Blob Data Reader" \ ---assignee-object-id "$(az identity show -n $IDENTITY_NAME -o tsv --query principalId -g $RESOURCE_GROUP)" \ ---scope "/subscriptions//resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" - -export IDENTITY_CLIENT_ID="$(az identity show -n ${IDENTITY_NAME} -g ${RESOURCE_GROUP} -otsv --query clientId)" -export IDENTITY_RESOURCE_ID="$(az identity show -n ${IDENTITY_NAME} -otsv --query id)" -``` - -Create an AzureIdentity object that references the identity created above: +**Note:** To use Object-Level Workload Identity (`.spec.serviceAccountName` with +cloud providers), the controller feature gate `ObjectLevelWorkloadIdentity` must +be enabled. ```yaml --- -apiVersion: aadpodidentity.k8s.io/v1 -kind: AzureIdentity -metadata: - name: # source-controller label will match this name - namespace: flux-system -spec: - clientID: - resourceID: - type: 0 # user-managed identity -``` - -Create an AzureIdentityBinding object that binds Pods with a specific selector -with the AzureIdentity created: - -```yaml -apiVersion: "aadpodidentity.k8s.io/v1" -kind: AzureIdentityBinding -metadata: - name: ${IDENTITY_NAME}-binding -spec: - azureIdentity: ${IDENTITY_NAME} - selector: ${IDENTITY_NAME} -``` - -Label the source-controller Deployment correctly so that it can match an identity binding: - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kustomize-controller - namespace: flux-system -spec: - template: - metadata: - labels: - aadpodidbinding: ${IDENTITY_NAME} # match the AzureIdentity name -``` - -If you have set up aad-pod-identity correctly and labeled the source-controller -Deployment, then you don't need to reference a Secret. - -```yaml apiVersion: source.toolkit.fluxcd.io/v1 kind: Bucket metadata: - name: azure-bucket - namespace: flux-system + name: azure-object-level-workload-identity + namespace: default spec: interval: 5m0s provider: azure - bucketName: testsas - endpoint: https://testfluxsas.blob.core.windows.net + bucketName: testwi + endpoint: https://testfluxwi.blob.core.windows.net + serviceAccountName: azure-workload-identity-sa + timeout: 30s +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-workload-identity-sa + namespace: default + annotations: + azure.workload.identity/client-id: + azure.workload.identity/tenant-id: ``` ##### Azure Blob SAS Token example diff --git a/internal/bucket/azure/blob.go b/internal/bucket/azure/blob.go index 24f778a8..5bf814b7 100644 --- a/internal/bucket/azure/blob.go +++ b/internal/bucket/azure/blob.go @@ -37,6 +37,8 @@ import ( corev1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" + "github.com/fluxcd/pkg/auth" + azureauth "github.com/fluxcd/pkg/auth/azure" "github.com/fluxcd/pkg/masktoken" sourcev1 "github.com/fluxcd/source-controller/api/v1" @@ -87,6 +89,7 @@ type options struct { proxyURL *url.URL withoutCredentials bool withoutRetries bool + authOpts []auth.Option } // withoutCredentials forces the BlobClient to not use any credentials. @@ -107,6 +110,13 @@ func withoutRetries() Option { } } +// WithAuth sets the auth options for workload identity authentication. +func WithAuth(authOpts ...auth.Option) Option { + return func(o *options) { + o.authOpts = authOpts + } +} + // NewClient creates a new Azure Blob storage client. // The credential config on the client is set based on the data from the // Bucket and Secret. It detects credentials in the Secret in the following @@ -130,7 +140,7 @@ func withoutRetries() Option { // // If no credentials are found, and the azidentity.ChainedTokenCredential can // not be established. A simple client without credentials is returned. -func NewClient(obj *sourcev1.Bucket, opts ...Option) (c *BlobClient, err error) { +func NewClient(ctx context.Context, obj *sourcev1.Bucket, opts ...Option) (c *BlobClient, err error) { c = &BlobClient{} var o options @@ -192,7 +202,7 @@ func NewClient(obj *sourcev1.Bucket, opts ...Option) (c *BlobClient, err error) // Compose token chain based on environment. // This functions as a replacement for azidentity.NewDefaultAzureCredential // to not shell out. - token, err = chainCredentialWithSecret(o.secret) + token, err = chainCredentialWithSecret(ctx, o.secret, o.authOpts...) if err != nil { err = fmt.Errorf("failed to create environment credential chain: %w", err) return nil, err @@ -470,7 +480,7 @@ func sasTokenFromSecret(ep string, secret *corev1.Secret) (string, error) { // - azidentity.ManagedIdentityCredential with defaults. // // If no valid token is created, it returns nil. -func chainCredentialWithSecret(secret *corev1.Secret) (azcore.TokenCredential, error) { +func chainCredentialWithSecret(ctx context.Context, secret *corev1.Secret, opts ...auth.Option) (azcore.TokenCredential, error) { var creds []azcore.TokenCredential credOpts := &azidentity.EnvironmentCredentialOptions{} @@ -483,28 +493,7 @@ func chainCredentialWithSecret(secret *corev1.Secret) (azcore.TokenCredential, e if token, _ := azidentity.NewEnvironmentCredential(credOpts); token != nil { creds = append(creds, token) } - if clientID := os.Getenv("AZURE_CLIENT_ID"); clientID != "" { - if file, ok := os.LookupEnv("AZURE_FEDERATED_TOKEN_FILE"); ok { - if _, ok := os.LookupEnv("AZURE_AUTHORITY_HOST"); ok { - if tenantID, ok := os.LookupEnv("AZURE_TENANT_ID"); ok { - if token, _ := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ - ClientID: clientID, - TenantID: tenantID, - TokenFilePath: file, - }); token != nil { - creds = append(creds, token) - } - } - } - } - - if token, _ := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ - ID: azidentity.ClientID(clientID), - }); token != nil { - creds = append(creds, token) - } - } - if token, _ := azidentity.NewManagedIdentityCredential(nil); token != nil { + if token := azureauth.NewTokenCredential(ctx, opts...); token != nil { creds = append(creds, token) } diff --git a/internal/bucket/azure/blob_test.go b/internal/bucket/azure/blob_test.go index 4fe82881..83f17e90 100644 --- a/internal/bucket/azure/blob_test.go +++ b/internal/bucket/azure/blob_test.go @@ -106,7 +106,8 @@ func TestNewClientAndBucketExistsWithProxy(t *testing.T) { }, } - client, err := NewClient(bucket, + client, err := NewClient(t.Context(), + bucket, WithProxyURL(tt.proxyURL), withoutCredentials(), withoutRetries()) @@ -472,7 +473,7 @@ func Test_sasTokenFromSecret(t *testing.T) { func Test_chainCredentialWithSecret(t *testing.T) { g := NewWithT(t) - got, err := chainCredentialWithSecret(nil) + got, err := chainCredentialWithSecret(t.Context(), nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(got).To(BeAssignableToTypeOf(&azidentity.ChainedTokenCredential{})) } diff --git a/internal/controller/bucket_controller.go b/internal/controller/bucket_controller.go index 8657d0e9..c855eac2 100644 --- a/internal/controller/bucket_controller.go +++ b/internal/controller/bucket_controller.go @@ -920,7 +920,8 @@ func (r *BucketReconciler) createBucketProvider(ctx context.Context, obj *source if creds.proxyURL != nil { opts = append(opts, azure.WithProxyURL(creds.proxyURL)) } - return azure.NewClient(obj, opts...) + opts = append(opts, azure.WithAuth(authOpts...)) + return azure.NewClient(ctx, obj, opts...) default: if err := minio.ValidateSecret(creds.secret); err != nil {