Azure Blob Storage v2 state store (#3203)

Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com>
Signed-off-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com>
Co-authored-by: Bernd Verst <github@bernd.dev>
This commit is contained in:
Alessandro (Ale) Segala 2023-11-02 16:19:22 -07:00 committed by GitHub
parent d6908e85e1
commit 0d488c2a9d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 329 additions and 99 deletions

View File

@ -509,8 +509,37 @@ const components = {
conformanceDestroy: 'conformance-state.aws.dynamodb-destroy.sh',
sourcePkg: 'state/aws/dynamodb',
},
'state.azure.blobstorage': {
'state.azure.blobstorage.v2': {
conformance: true,
requiredSecrets: [
'AzureBlobStorageAccount',
'AzureBlobStorageAccessKey',
'AzureCertificationTenantId',
'AzureCertificationServicePrincipalClientId',
'AzureCertificationServicePrincipalClientSecret',
'AzureBlobStorageContainer',
],
sourcePkg: [
'state/azure/blobstorage',
'internal/component/azure/blobstorage',
],
},
'state.azure.blobstorage.v1': {
conformance: true,
requiredSecrets: [
'AzureBlobStorageAccount',
'AzureBlobStorageAccessKey',
'AzureCertificationTenantId',
'AzureCertificationServicePrincipalClientId',
'AzureCertificationServicePrincipalClientSecret',
'AzureBlobStorageContainer',
],
sourcePkg: [
'state/azure/blobstorage',
'internal/component/azure/blobstorage',
],
},
'state.azure.blobstorage': {
certification: true,
requiredSecrets: [
'AzureBlobStorageAccount',

View File

@ -1,5 +1,5 @@
/*
Copyright 2021 The Dapr Authors
Copyright 2023 The Dapr Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
@ -11,36 +11,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Azure Blob Storage state store.
Sample configuration in yaml:
apiVersion: dapr.io/v1alpha1
kind: Component
metadata:
name: statestore
spec:
type: state.azure.blobstorage
metadata:
- name: accountName
value: <storage account name>
- name: accountKey
value: <key>
- name: containerName
value: <container Name>
Concurrency is supported with ETags according to https://docs.microsoft.com/en-us/azure/storage/common/storage-concurrency#managing-concurrency-in-blob-storage
*/
package blobstorage
package internal
import (
"context"
"fmt"
"io"
"reflect"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
@ -56,18 +33,24 @@ import (
"github.com/dapr/kit/ptr"
)
const (
keyDelimiter = "||"
)
// StateStore Type.
type StateStore struct {
state.BulkStore
getFileNameFn func(string) string
containerClient *container.Client
logger logger.Logger
}
func NewAzureBlobStorageStore(logger logger.Logger, getFileNameFn func(string) string) state.Store {
s := &StateStore{
logger: logger,
getFileNameFn: getFileNameFn,
}
s.BulkStore = state.NewDefaultBulkStore(s)
return s
}
// Init the connection to blob storage, optionally creates a blob container if it doesn't exist.
func (r *StateStore) Init(ctx context.Context, metadata state.Metadata) error {
var err error
@ -100,7 +83,7 @@ func (r *StateStore) Set(ctx context.Context, req *state.SetRequest) error {
func (r *StateStore) Ping(ctx context.Context) error {
if _, err := r.containerClient.GetProperties(ctx, nil); err != nil {
return fmt.Errorf("blob storage: error connecting to Blob storage at %s: %s", r.containerClient.URL(), err)
return fmt.Errorf("error connecting to Azure Blob Storage at '%s': %w", r.containerClient.URL(), err)
}
return nil
@ -112,17 +95,8 @@ func (r *StateStore) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) {
return
}
// NewAzureBlobStorageStore instance.
func NewAzureBlobStorageStore(logger logger.Logger) state.Store {
s := &StateStore{
logger: logger,
}
s.BulkStore = state.NewDefaultBulkStore(s)
return s
}
func (r *StateStore) readFile(ctx context.Context, req *state.GetRequest) (*state.GetResponse, error) {
blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key))
blockBlobClient := r.containerClient.NewBlockBlobClient(r.getFileNameFn(req.Key))
blobDownloadResponse, err := blockBlobClient.DownloadStream(ctx, nil)
if err != nil {
if isNotFoundError(err) {
@ -132,19 +106,16 @@ func (r *StateStore) readFile(ctx context.Context, req *state.GetRequest) (*stat
return &state.GetResponse{}, err
}
reader := blobDownloadResponse.Body
defer reader.Close()
blobData, err := io.ReadAll(reader)
defer blobDownloadResponse.Body.Close()
blobData, err := io.ReadAll(blobDownloadResponse.Body)
if err != nil {
return &state.GetResponse{}, fmt.Errorf("error reading az blob: %w", err)
return &state.GetResponse{}, fmt.Errorf("error reading blob: %w", err)
}
contentType := blobDownloadResponse.ContentType
return &state.GetResponse{
Data: blobData,
ETag: ptr.Of(string(*blobDownloadResponse.ETag)),
ContentType: contentType,
ContentType: blobDownloadResponse.ContentType,
}, nil
}
@ -158,22 +129,20 @@ func (r *StateStore) writeFile(ctx context.Context, req *state.SetRequest) error
modifiedAccessConditions.IfNoneMatch = ptr.Of(azcore.ETagAny)
}
accessConditions := blob.AccessConditions{
ModifiedAccessConditions: &modifiedAccessConditions,
}
blobHTTPHeaders, err := storageinternal.CreateBlobHTTPHeadersFromRequest(req.Metadata, req.ContentType, r.logger)
if err != nil {
return err
}
uploadOptions := azblob.UploadBufferOptions{
AccessConditions: &accessConditions,
Metadata: storageinternal.SanitizeMetadata(r.logger, req.Metadata),
HTTPHeaders: &blobHTTPHeaders,
AccessConditions: &blob.AccessConditions{
ModifiedAccessConditions: &modifiedAccessConditions,
},
Metadata: storageinternal.SanitizeMetadata(r.logger, req.Metadata),
HTTPHeaders: &blobHTTPHeaders,
}
blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key))
blockBlobClient := r.containerClient.NewBlockBlobClient(r.getFileNameFn(req.Key))
_, err = blockBlobClient.UploadBuffer(ctx, r.marshal(req), &uploadOptions)
if err != nil {
@ -182,14 +151,14 @@ func (r *StateStore) writeFile(ctx context.Context, req *state.SetRequest) error
return state.NewETagError(state.ETagMismatch, err)
}
return fmt.Errorf("error uploading az blob: %w", err)
return fmt.Errorf("error uploading blob: %w", err)
}
return nil
}
func (r *StateStore) deleteFile(ctx context.Context, req *state.DeleteRequest) error {
blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key))
blockBlobClient := r.containerClient.NewBlockBlobClient(r.getFileNameFn(req.Key))
modifiedAccessConditions := blob.ModifiedAccessConditions{}
if req.HasETag() {
@ -218,25 +187,13 @@ func (r *StateStore) deleteFile(ctx context.Context, req *state.DeleteRequest) e
return nil
}
func getFileName(key string) string {
pr := strings.Split(key, keyDelimiter)
if len(pr) != 2 {
return pr[0]
}
return pr[1]
}
func (r *StateStore) marshal(req *state.SetRequest) []byte {
var v string
b, ok := req.Value.([]byte)
if ok {
v = string(b)
} else {
v, _ = jsoniter.MarshalToString(req.Value)
if !ok {
b, _ = jsoniter.Marshal(req.Value)
}
return []byte(v)
return b
}
func isNotFoundError(err error) bool {

View File

@ -11,7 +11,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package blobstorage
package internal
import (
"context"
@ -39,15 +39,3 @@ func TestInit(t *testing.T) {
assert.Equal(t, err, fmt.Errorf("missing or empty accountName field from metadata"))
})
}
func TestFileName(t *testing.T) {
t.Run("Valid composite key", func(t *testing.T) {
key := getFileName("app_id||key")
assert.Equal(t, "key", key)
})
t.Run("No delimiter present", func(t *testing.T) {
key := getFileName("key")
assert.Equal(t, "key", key)
})
}

View File

@ -1,4 +1,4 @@
# yaml-language-server: $schema=../../../component-metadata-schema.json
# yaml-language-server: $schema=../../../../component-metadata-schema.json
schemaVersion: v1
type: state
name: azure.blobstorage

View File

@ -0,0 +1,42 @@
/*
Copyright 2023 The Dapr Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package blobstorage
import (
"strings"
"github.com/dapr/components-contrib/state"
"github.com/dapr/components-contrib/state/azure/blobstorage/internal"
"github.com/dapr/kit/logger"
)
const (
keyDelimiter = "||"
)
func getFileName(key string) string {
// This function splits the prefix, such as the Dapr app ID, from the key.
// This is wrong, as state stores should honor the prefix, but it is kept here for backwards-compatibility.
// The v2 of the component fixes thie behavior.
pr := strings.Split(key, keyDelimiter)
if len(pr) != 2 {
return pr[0]
}
return pr[1]
}
func NewAzureBlobStorageStore(log logger.Logger) state.Store {
return internal.NewAzureBlobStorageStore(log, getFileName)
}

View File

@ -0,0 +1,32 @@
/*
Copyright 2023 The Dapr Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package blobstorage
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestFileName(t *testing.T) {
t.Run("Valid composite key", func(t *testing.T) {
key := getFileName("app_id||key")
assert.Equal(t, "key", key)
})
t.Run("No delimiter present", func(t *testing.T) {
key := getFileName("key")
assert.Equal(t, "key", key)
})
}

View File

@ -0,0 +1,76 @@
# yaml-language-server: $schema=../../../../component-metadata-schema.json
schemaVersion: v1
type: state
name: azure.blobstorage
version: v2
status: stable
title: "Azure Blob Storage"
urls:
- title: Reference
url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-azure-blobstorage/
capabilities:
- crud
- etag
builtinAuthenticationProfiles:
- name: "azuread"
metadata:
- name: accountName
required: true
sensitive: false
description: "The storage account name"
example: '"mystorageaccount"'
authenticationProfiles:
- title: "Connection string"
description: "Authenticate using a connection string."
metadata:
- name: connectionString
required: true
sensitive: true
description: "Shared access policy connection string for Blob Storage."
example: '"BlobEndpoint=https://storagesample.blob.core.windows.net;SharedAccessSignature={KeySig}"'
- title: "Account Key"
description: "Authenticate using a pre-shared \"account key\"."
metadata:
- name: accountKey
required: true
sensitive: true
description: "The key to authenticate to the Storage Account."
example: '"my-secret-key"'
- name: accountName
required: true
sensitive: false
description: "The storage account name"
example: '"mystorageaccount"'
metadata:
- name: containerName
required: true
sensitive: false
description: |
The name of the container to be used for Dapr state. The container will be created for you if it doesn't exist.
example: '"container"'
- name: publicAccessLevel
type: string
required: false
description: |
Indicates whether data in the container may be accessed publicly and the level of access.
example: '"none"'
default: '"none"'
allowedValues:
- "none" # Equivalent to private - https://learn.microsoft.com/en-us/rest/api/storageservices/get-container-acl
- "blob"
- "container"
- name: endpoint
type: string
description: |
Optional custom endpoint URL. This is useful when using the Azurite emulator or when using custom domains for Azure Storage (although this is not officially supported). The endpoint must be the full base URL, including the protocol (http:// or https://), the IP or FQDN, and optional port.
required: false
example: '"http://127.0.0.1:10000"'
- name: retryCount
# getBlobRetryCount is a deprecated alias for this field
type: number
required: false
default: "3"
example: "3"
description: |
Specifies the maximum number of HTTP requests that will be made to retry blob operations.
A value of zero means that no additional attempts will be made after a failure.

View File

@ -0,0 +1,29 @@
/*
Copyright 2023 The Dapr Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package blobstorage
import (
"github.com/dapr/components-contrib/state"
"github.com/dapr/components-contrib/state/azure/blobstorage/internal"
"github.com/dapr/kit/logger"
)
func getFileName(key string) string {
// Nop in this version
return key
}
func NewAzureBlobStorageStore(log logger.Logger) state.Store {
return internal.NewAzureBlobStorageStore(log, getFileName)
}

View File

@ -20,9 +20,11 @@ import (
"strconv"
"testing"
blob "github.com/dapr/components-contrib/state/azure/blobstorage"
component_v1 "github.com/dapr/components-contrib/state/azure/blobstorage/v1"
component_v2 "github.com/dapr/components-contrib/state/azure/blobstorage/v2"
"github.com/dapr/components-contrib/tests/certification/embedded"
"github.com/dapr/components-contrib/tests/certification/flow"
"github.com/dapr/dapr/pkg/components"
"github.com/dapr/go-sdk/client"
"github.com/dapr/kit/logger"
@ -78,14 +80,27 @@ func TestAzureBlobStorage(t *testing.T) {
return nil
}
flow.New(t, "Test basic operations, save/get/delete using existing container").
flow.New(t, "v1 - Test basic operations, save/get/delete using existing container").
// Run the Dapr sidecar with azure blob storage.
Step(sidecar.Run(sidecarNamePrefix,
append(componentRuntimeOptions(),
embedded.WithoutApp(),
embedded.WithDaprGRPCPort(strconv.Itoa(currentGrpcPort)),
embedded.WithDaprHTTPPort(strconv.Itoa(currentHTTPPort)),
embedded.WithComponentsPath("./components/basictest"),
embedded.WithResourcesPath("./components/basictest-v1"),
)...,
)).
Step("Run basic test with existing container", basicTest("statestore-basic")).
Run()
flow.New(t, "v2 - Test basic operations, save/get/delete using existing container").
// Run the Dapr sidecar with azure blob storage.
Step(sidecar.Run(sidecarNamePrefix,
append(componentRuntimeOptions(),
embedded.WithoutApp(),
embedded.WithDaprGRPCPort(strconv.Itoa(currentGrpcPort)),
embedded.WithDaprHTTPPort(strconv.Itoa(currentHTTPPort)),
embedded.WithResourcesPath("./components/basictest-v2"),
)...,
)).
Step("Run basic test with existing container", basicTest("statestore-basic")).
@ -98,11 +113,11 @@ func TestAzureBlobStorage(t *testing.T) {
embedded.WithoutApp(),
embedded.WithDaprGRPCPort(strconv.Itoa(currentGrpcPort)),
embedded.WithDaprHTTPPort(strconv.Itoa(currentHTTPPort)),
embedded.WithComponentsPath("./components/nonexistingcontainertest"),
embedded.WithResourcesPath("./components/nonexistingcontainertest"),
)...,
)).
Step("Run basic test with new table", basicTest("statestore-newcontainer")).
Step("Delete the New Table", deleteContainer).
Step("Run basic test with new container", basicTest("statestore-newcontainer")).
Step("Delete the new container", deleteContainer).
Run()
flow.New(t, "Test for authentication using Azure Auth layer").
@ -112,7 +127,7 @@ func TestAzureBlobStorage(t *testing.T) {
embedded.WithoutApp(),
embedded.WithDaprGRPCPort(strconv.Itoa(currentGrpcPort)),
embedded.WithDaprHTTPPort(strconv.Itoa(currentHTTPPort)),
embedded.WithComponentsPath("./components/aadtest"),
embedded.WithResourcesPath("./components/aadtest"),
)...,
)).
Step("Run AAD test", basicTest("statestore-aad")).
@ -124,7 +139,15 @@ func componentRuntimeOptions() []embedded.Option {
stateRegistry := state_loader.NewRegistry()
stateRegistry.Logger = log
stateRegistry.RegisterComponent(blob.NewAzureBlobStorageStore, "azure.blobstorage")
stateRegistry.RegisterComponentWithVersions("azure.blobstorage", components.Versioning{
Preferred: components.VersionConstructor{
Version: "v2", Constructor: component_v1.NewAzureBlobStorageStore,
},
Deprecated: []components.VersionConstructor{
{Version: "v1", Constructor: component_v2.NewAzureBlobStorageStore},
},
Default: "v1",
})
secretstoreRegistry := secretstores_loader.NewRegistry()
secretstoreRegistry.Logger = log

View File

@ -4,6 +4,7 @@ metadata:
name: statestore-aad
spec:
type: state.azure.blobstorage
version: v2
metadata:
- name: accountName
secretKeyRef:

View File

@ -4,6 +4,7 @@ metadata:
name: statestore-basic
spec:
type: state.azure.blobstorage
version: v1
metadata:
- name: accountName
secretKeyRef:

View File

@ -0,0 +1,22 @@
apiVersion: dapr.io/v1alpha1
kind: Component
metadata:
name: statestore-basic
spec:
type: state.azure.blobstorage
version: v2
metadata:
- name: accountName
secretKeyRef:
name: AzureBlobStorageAccount
key: AzureBlobStorageAccount
- name: accountKey
secretKeyRef:
name: AzureBlobStorageAccessKey
key: AzureBlobStorageAccessKey
- name: containerName
secretKeyRef:
name: AzureBlobStorageContainer
key: AzureBlobStorageContainer
auth:
secretstore: envvar-secret-store

View File

@ -0,0 +1,9 @@
apiVersion: dapr.io/v1alpha1
kind: Component
metadata:
name: envvar-secret-store
namespace: default
spec:
type: secretstores.local.env
version: v1
metadata:

View File

@ -4,6 +4,7 @@ metadata:
name: statestore-newcontainer
spec:
type: state.azure.blobstorage
version: v2
metadata:
- name: accountName
secretKeyRef:

View File

@ -4,6 +4,7 @@ metadata:
name: statestore
spec:
type: state.azure.blobstorage
version: v1
metadata:
- name: accountName
value: ${{AzureBlobStorageAccount}}

View File

@ -0,0 +1,14 @@
apiVersion: dapr.io/v1alpha1
kind: Component
metadata:
name: statestore
spec:
type: state.azure.blobstorage
version: v2
metadata:
- name: accountName
value: ${{AzureBlobStorageAccount}}
- name: accountKey
value: ${{AzureBlobStorageAccessKey}}
- name: containerName
value: statestore-conf

View File

@ -20,7 +20,9 @@ components:
operations: [ "ttl" ]
- component: azure.cosmosdb
operations: [ "transaction", "etag", "first-write", "query", "ttl" ]
- component: azure.blobstorage
- component: azure.blobstorage.v1
operations: [ "etag", "first-write" ]
- component: azure.blobstorage.v2
operations: [ "etag", "first-write" ]
- component: azure.sql
operations: [ "transaction", "etag", "first-write", "ttl" ]

View File

@ -24,7 +24,8 @@ import (
"github.com/dapr/components-contrib/state"
s_awsdynamodb "github.com/dapr/components-contrib/state/aws/dynamodb"
s_blobstorage "github.com/dapr/components-contrib/state/azure/blobstorage"
s_blobstorage_v1 "github.com/dapr/components-contrib/state/azure/blobstorage/v1"
s_blobstorage_v2 "github.com/dapr/components-contrib/state/azure/blobstorage/v2"
s_cosmosdb "github.com/dapr/components-contrib/state/azure/cosmosdb"
s_azuretablestorage "github.com/dapr/components-contrib/state/azure/tablestorage"
s_cassandra "github.com/dapr/components-contrib/state/cassandra"
@ -78,8 +79,10 @@ func loadStateStore(name string) state.Store {
return s_redis.NewRedisStateStore(testLogger)
case "redis.v7":
return s_redis.NewRedisStateStore(testLogger)
case "azure.blobstorage":
return s_blobstorage.NewAzureBlobStorageStore(testLogger)
case "azure.blobstorage.v1":
return s_blobstorage_v1.NewAzureBlobStorageStore(testLogger)
case "azure.blobstorage.v2":
return s_blobstorage_v2.NewAzureBlobStorageStore(testLogger)
case "azure.cosmosdb":
return s_cosmosdb.NewCosmosDBStateStore(testLogger)
case "mongodb":