Create the storage container if it doesn't exist (to preserve compatibility with old SDK's behavior)
Also fixes to tests Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com>
This commit is contained in:
ItalyPaleAle
parent
c76526c805
commit
a0251482c2
|
|
@ -26,6 +26,8 @@ import (
|
|||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
azauth "github.com/dapr/components-contrib/internal/authentication/azure"
|
||||
|
|
@ -486,7 +488,7 @@ func (aeh *AzureEventHubs) getProducerClientForTopic(ctx context.Context, topic
|
|||
// Creates a processor for a given topic.
|
||||
func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic string) (*azeventhubs.Processor, error) {
|
||||
// Get the checkpoint store
|
||||
checkpointStore, err := aeh.getCheckpointStore()
|
||||
checkpointStore, err := aeh.getCheckpointStore(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err)
|
||||
}
|
||||
|
|
@ -546,7 +548,7 @@ func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic strin
|
|||
}
|
||||
|
||||
// Returns the checkpoint store from the object. If it doesn't exist, it lazily initializes it.
|
||||
func (aeh *AzureEventHubs) getCheckpointStore() (azeventhubs.CheckpointStore, error) {
|
||||
func (aeh *AzureEventHubs) getCheckpointStore(ctx context.Context) (azeventhubs.CheckpointStore, error) {
|
||||
// Check if we have the checkpoint store
|
||||
aeh.checkpointStoreLock.RLock()
|
||||
if aeh.checkpointStoreCache != nil {
|
||||
|
|
@ -565,7 +567,7 @@ func (aeh *AzureEventHubs) getCheckpointStore() (azeventhubs.CheckpointStore, er
|
|||
|
||||
// Init the checkpoint store and store it in the object
|
||||
var err error
|
||||
aeh.checkpointStoreCache, err = aeh.createCheckpointStore()
|
||||
aeh.checkpointStoreCache, err = aeh.createCheckpointStore(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err)
|
||||
}
|
||||
|
|
@ -574,7 +576,7 @@ func (aeh *AzureEventHubs) getCheckpointStore() (azeventhubs.CheckpointStore, er
|
|||
}
|
||||
|
||||
// Initializes a new checkpoint store
|
||||
func (aeh *AzureEventHubs) createCheckpointStore() (checkpointStore azeventhubs.CheckpointStore, err error) {
|
||||
func (aeh *AzureEventHubs) createCheckpointStore(ctx context.Context) (checkpointStore azeventhubs.CheckpointStore, err error) {
|
||||
if aeh.metadata.StorageAccountName == "" {
|
||||
return nil, errors.New("property storageAccountName is required to subscribe to an Event Hub topic")
|
||||
}
|
||||
|
|
@ -582,6 +584,13 @@ func (aeh *AzureEventHubs) createCheckpointStore() (checkpointStore azeventhubs.
|
|||
return nil, errors.New("property storageContainerName is required to subscribe to an Event Hub topic")
|
||||
}
|
||||
|
||||
// Ensure the container exists
|
||||
err = aeh.ensureStorageContainer(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create the checkpoint store
|
||||
checkpointStoreOpts := &checkpoints.BlobStoreOptions{
|
||||
ClientOptions: policy.ClientOptions{
|
||||
Telemetry: policy.TelemetryOptions{
|
||||
|
|
@ -620,6 +629,89 @@ func (aeh *AzureEventHubs) createCheckpointStore() (checkpointStore azeventhubs.
|
|||
return checkpointStore, nil
|
||||
}
|
||||
|
||||
// Ensures that the container exists in the Azure Storage Account.
|
||||
// This is done to preserve backwards-compatibility with Dapr 1.9, as the old checkpoint SDK created them automatically.
|
||||
func (aeh *AzureEventHubs) ensureStorageContainer(parentCtx context.Context) error {
|
||||
// Get a client to Azure Blob Storage
|
||||
client, err := aeh.createStorageClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the container
|
||||
// This will return an error if it already exists
|
||||
ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout)
|
||||
defer cancel()
|
||||
_, err = client.CreateContainer(ctx, aeh.metadata.StorageContainerName, &container.CreateOptions{
|
||||
// Default is private
|
||||
Access: nil,
|
||||
})
|
||||
if err != nil {
|
||||
// Check if it's an Azure Storage error
|
||||
resErr := &azcore.ResponseError{}
|
||||
// If the container already exists, return no error
|
||||
if errors.As(err, &resErr) && (resErr.ErrorCode == "ContainerAlreadyExists" || resErr.ErrorCode == "ResourceAlreadyExists") {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to create Azure Storage container %s: %w", aeh.metadata.StorageContainerName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Creates a client to access Azure Blob Storage
|
||||
func (aeh *AzureEventHubs) createStorageClient() (*azblob.Client, error) {
|
||||
options := azblob.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Telemetry: policy.TelemetryOptions{
|
||||
ApplicationID: "dapr-" + logger.DaprVersion,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var (
|
||||
err error
|
||||
client *azblob.Client
|
||||
)
|
||||
// Use the global URL for Azure Storage
|
||||
accountURL := fmt.Sprintf("https://%s.blob.%s", aeh.metadata.StorageAccountName, "core.windows.net")
|
||||
|
||||
if aeh.metadata.StorageConnectionString != "" {
|
||||
// Authenticate with a connection string
|
||||
client, err = azblob.NewClientFromConnectionString(aeh.metadata.StorageConnectionString, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating Azure Storage client from connection string: %w", err)
|
||||
}
|
||||
} else if aeh.metadata.StorageAccountKey != "" {
|
||||
// Authenticate with a shared key
|
||||
credential, newSharedKeyErr := azblob.NewSharedKeyCredential(aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey)
|
||||
if newSharedKeyErr != nil {
|
||||
return nil, fmt.Errorf("invalid Azure Storage shared key credentials with error: %w", newSharedKeyErr)
|
||||
}
|
||||
client, err = azblob.NewClientWithSharedKeyCredential(accountURL, credential, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating Azure Storage client from shared key credentials: %w", err)
|
||||
}
|
||||
} else {
|
||||
// Use Azure AD
|
||||
settings, err := azauth.NewEnvironmentSettings("storage", aeh.metadata.properties)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
credential, tokenErr := settings.GetTokenCredential()
|
||||
if tokenErr != nil {
|
||||
return nil, fmt.Errorf("invalid Azure Storage token credentials with error: %w", tokenErr)
|
||||
}
|
||||
client, err = azblob.NewClient(accountURL, credential, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating Azure Storage client from token credentials: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// Returns a connection string with the Event Hub name (entity path) set if not present.
|
||||
func (aeh *AzureEventHubs) constructConnectionStringFromTopic(topic string) (string, error) {
|
||||
if aeh.metadata.hubName != "" {
|
||||
|
|
|
|||
|
|
@ -17,16 +17,13 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
|
||||
azauth "github.com/dapr/components-contrib/internal/authentication/azure"
|
||||
"github.com/dapr/kit/logger"
|
||||
"github.com/dapr/kit/retry"
|
||||
)
|
||||
|
||||
|
|
@ -37,7 +34,7 @@ import (
|
|||
// TODO(@ItalyPaleAle): Remove this for Dapr 1.13
|
||||
func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, topic string) error {
|
||||
// Get a client to Azure Blob Storage
|
||||
client, err := aeh.createContainerStorageClient()
|
||||
client, err := aeh.createStorageClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -55,7 +52,7 @@ func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context,
|
|||
backOffConfig.MaxRetries = -1
|
||||
b := backOffConfig.NewBackOffWithContext(parentCtx)
|
||||
err = backoff.Retry(func() error {
|
||||
pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
|
||||
pager := client.NewListBlobsFlatPager(aeh.metadata.StorageContainerName, &container.ListBlobsFlatOptions{
|
||||
Prefix: &prefix,
|
||||
})
|
||||
for pager.More() {
|
||||
|
|
@ -64,6 +61,12 @@ func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context,
|
|||
cancel()
|
||||
if innerErr != nil {
|
||||
// Treat these errors as permanent
|
||||
resErr := &azcore.ResponseError{}
|
||||
if !errors.As(err, &resErr) || resErr.StatusCode != http.StatusNotFound {
|
||||
// A "not-found" error means that the storage container doesn't exist, so let's not handle it here
|
||||
// Just return no error
|
||||
return nil
|
||||
}
|
||||
return backoff.Permanent(fmt.Errorf("failed to list blobs: %w", innerErr))
|
||||
}
|
||||
for _, blob := range resp.Segment.BlobItems {
|
||||
|
|
@ -87,54 +90,3 @@ func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context,
|
|||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (aeh *AzureEventHubs) createContainerStorageClient() (*container.Client, error) {
|
||||
options := container.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Telemetry: policy.TelemetryOptions{
|
||||
ApplicationID: "dapr-" + logger.DaprVersion,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var (
|
||||
err error
|
||||
client *container.Client
|
||||
)
|
||||
// Use the global URL for Azure Storage
|
||||
containerURL := fmt.Sprintf("https://%s.blob.%s/%s", aeh.metadata.StorageAccountName, "core.windows.net", aeh.metadata.StorageContainerName)
|
||||
|
||||
if aeh.metadata.StorageConnectionString != "" {
|
||||
// Authenticate with a connection string
|
||||
client, err = container.NewClientFromConnectionString(aeh.metadata.StorageConnectionString, aeh.metadata.StorageContainerName, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating Azure Storage client from connection string: %w", err)
|
||||
}
|
||||
} else if aeh.metadata.StorageAccountKey != "" {
|
||||
// Authenticate with a shared key
|
||||
credential, newSharedKeyErr := azblob.NewSharedKeyCredential(aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey)
|
||||
if newSharedKeyErr != nil {
|
||||
return nil, fmt.Errorf("invalid Azure Storage shared key credentials with error: %w", newSharedKeyErr)
|
||||
}
|
||||
client, err = container.NewClientWithSharedKeyCredential(containerURL, credential, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating Azure Storage client from shared key credentials: %w", err)
|
||||
}
|
||||
} else {
|
||||
// Use Azure AD
|
||||
settings, err := azauth.NewEnvironmentSettings("storage", aeh.metadata.properties)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
credential, tokenErr := settings.GetTokenCredential()
|
||||
if tokenErr != nil {
|
||||
return nil, fmt.Errorf("invalid Azure Storage token credentials with error: %w", tokenErr)
|
||||
}
|
||||
client, err = container.NewClient(containerURL, credential, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating Azure Storage client from token credentials: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ import (
|
|||
"go.uber.org/multierr"
|
||||
|
||||
// Pub-Sub.
|
||||
|
||||
pubsub_evethubs "github.com/dapr/components-contrib/pubsub/azure/eventhubs"
|
||||
"github.com/dapr/components-contrib/pubsub"
|
||||
pubsub_eventhubs "github.com/dapr/components-contrib/pubsub/azure/eventhubs"
|
||||
secretstore_env "github.com/dapr/components-contrib/secretstores/local/env"
|
||||
pubsub_loader "github.com/dapr/dapr/pkg/components/pubsub"
|
||||
secretstores_loader "github.com/dapr/dapr/pkg/components/secretstores"
|
||||
|
|
@ -131,7 +131,7 @@ func TestEventhubs(t *testing.T) {
|
|||
client := sidecar.GetClient(ctx, sidecarName)
|
||||
|
||||
// publish messages
|
||||
ctx.Logf("Publishing messages. sidecarName: %s, topicName: %s", sidecarName, topicName)
|
||||
logs := fmt.Sprintf("Published messages. sidecarName: %s, topicName: %s", sidecarName, topicName)
|
||||
|
||||
var publishOptions dapr.PublishEventOption
|
||||
|
||||
|
|
@ -139,8 +139,8 @@ func TestEventhubs(t *testing.T) {
|
|||
publishOptions = dapr.PublishEventWithMetadata(metadata)
|
||||
}
|
||||
|
||||
for _, message := range messages {
|
||||
ctx.Logf("Publishing: %q", message)
|
||||
for i, message := range messages {
|
||||
logs += fmt.Sprintf("\nMessage %d: %s", i, message)
|
||||
var err error
|
||||
|
||||
if publishOptions != nil {
|
||||
|
|
@ -149,8 +149,11 @@ func TestEventhubs(t *testing.T) {
|
|||
err = client.PublishEvent(ctx, pubsubName, topicName, message)
|
||||
}
|
||||
|
||||
require.NoError(ctx, err, "error publishing message")
|
||||
require.NoErrorf(ctx, err, "error publishing message %s", message)
|
||||
}
|
||||
|
||||
ctx.Log(logs)
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
@ -159,7 +162,7 @@ func TestEventhubs(t *testing.T) {
|
|||
return func(ctx flow.Context) error {
|
||||
// assert for messages
|
||||
for _, m := range messageWatchers {
|
||||
m.Assert(ctx, 25*timeout)
|
||||
m.Assert(ctx, 10*timeout)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -181,13 +184,19 @@ func TestEventhubs(t *testing.T) {
|
|||
messageWatchers.ExpectStrings(messages...)
|
||||
|
||||
output, err := exec.Command("/bin/sh", "send-iot-device-events.sh", topicToBeCreated).Output()
|
||||
assert.Nil(t, err, "Error in send-iot-device-events.sh.:\n%s", string(output))
|
||||
assert.NoErrorf(t, err, "Error in send-iot-device-events.sh.:\n%s", string(output))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// Topic name for a IOT device is same as IOTHubName
|
||||
iotHubName := os.Getenv(iotHubNameEnvKey)
|
||||
|
||||
// Here so we can comment out tests as needed
|
||||
_ = publishMessageAsDevice
|
||||
_ = iotHubName
|
||||
_ = consumerGroup4
|
||||
_ = consumerGroup5
|
||||
|
||||
flow.New(t, "eventhubs certification").
|
||||
|
||||
// Test : single publisher, multiple subscriber with their own consumerID
|
||||
|
|
@ -201,7 +210,8 @@ func TestEventhubs(t *testing.T) {
|
|||
embedded.WithAppProtocol(runtime.HTTPProtocol, appPort),
|
||||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort),
|
||||
componentRuntimeOptions(),
|
||||
embedded.WithProfilePort(runtime.DefaultProfilePort),
|
||||
componentRuntimeOptions(1),
|
||||
)).
|
||||
|
||||
// Run subscriberApplication app2
|
||||
|
|
@ -215,8 +225,9 @@ func TestEventhubs(t *testing.T) {
|
|||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset),
|
||||
embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset),
|
||||
componentRuntimeOptions(),
|
||||
componentRuntimeOptions(2),
|
||||
)).
|
||||
Step("wait", flow.Sleep(10*time.Second)).
|
||||
Step("publish messages to topic1", publishMessages(nil, sidecarName1, topicActiveName, consumerGroup1, consumerGroup2)).
|
||||
Step("publish messages to unUsedTopic", publishMessages(nil, sidecarName1, topicPassiveName)).
|
||||
Step("verify if app1 has recevied messages published to topic1", assertMessages(10*time.Second, consumerGroup1)).
|
||||
|
|
@ -235,13 +246,15 @@ func TestEventhubs(t *testing.T) {
|
|||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*2),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*2),
|
||||
embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*2),
|
||||
componentRuntimeOptions(),
|
||||
componentRuntimeOptions(3),
|
||||
)).
|
||||
Step("wait", flow.Sleep(10*time.Second)).
|
||||
|
||||
// publish message in topic1 from two publisher apps, however there are two subscriber apps (app2,app3) with same consumerID
|
||||
Step("publish messages to topic1", publishMessages(metadata, sidecarName1, topicActiveName, consumerGroup2)).
|
||||
Step("publish messages to topic1", publishMessages(metadata1, sidecarName2, topicActiveName, consumerGroup2)).
|
||||
Step("publish messages to topic1 from app1", publishMessages(metadata, sidecarName1, topicActiveName, consumerGroup2)).
|
||||
Step("publish messages to topic1 from app2", publishMessages(metadata1, sidecarName2, topicActiveName, consumerGroup2)).
|
||||
Step("verify if app2, app3 together have recevied messages published to topic1", assertMessages(10*time.Second, consumerGroup2)).
|
||||
|
||||
// Test : Entitymanagement , Test partition key, in order processing with single publisher/subscriber
|
||||
// Run subscriberApplication app4
|
||||
Step(app.Run(appID4, fmt.Sprintf(":%d", appPort+portOffset*3),
|
||||
|
|
@ -254,7 +267,7 @@ func TestEventhubs(t *testing.T) {
|
|||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3),
|
||||
embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*3),
|
||||
componentRuntimeOptions(),
|
||||
componentRuntimeOptions(4),
|
||||
)).
|
||||
Step(fmt.Sprintf("publish messages to topicToBeCreated: %s", topicToBeCreated), publishMessages(metadata, sidecarName4, topicToBeCreated, consumerGroup4)).
|
||||
Step("verify if app4 has recevied messages published to newly created topic", assertMessages(10*time.Second, consumerGroup4)).
|
||||
|
|
@ -270,22 +283,31 @@ func TestEventhubs(t *testing.T) {
|
|||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*4),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*4),
|
||||
embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*4),
|
||||
componentRuntimeOptions(),
|
||||
componentRuntimeOptions(5),
|
||||
)).
|
||||
Step("add expected IOT messages (simulate add message to iot)", publishMessageAsDevice(consumerGroup5)).
|
||||
Step("verify if app5 has recevied messages published to iot topic", assertMessages(40*time.Second, consumerGroup5)).
|
||||
Step("wait", flow.Sleep(5*time.Second)).
|
||||
|
||||
// cleanup azure assets created as part of tests
|
||||
Step("delete eventhub created as part of the eventhub management test", deleteEventhub).
|
||||
Run()
|
||||
}
|
||||
|
||||
func componentRuntimeOptions() []runtime.Option {
|
||||
func componentRuntimeOptions(instance int) []runtime.Option {
|
||||
log := logger.NewLogger("dapr.components")
|
||||
log.SetOutputLevel(logger.DebugLevel)
|
||||
|
||||
pubsubRegistry := pubsub_loader.NewRegistry()
|
||||
pubsubRegistry.Logger = log
|
||||
pubsubRegistry.RegisterComponent(pubsub_evethubs.NewAzureEventHubs, "azure.eventhubs")
|
||||
pubsubRegistry.RegisterComponent(func(l logger.Logger) pubsub.PubSub {
|
||||
l = l.WithFields(map[string]any{
|
||||
"component": "pubsub.azure.eventhubs",
|
||||
"instance": instance,
|
||||
})
|
||||
l.Infof("Instantiated log for instance %d", instance)
|
||||
return pubsub_eventhubs.NewAzureEventHubs(l)
|
||||
}, "azure.eventhubs")
|
||||
|
||||
secretstoreRegistry := secretstores_loader.NewRegistry()
|
||||
secretstoreRegistry.Logger = log
|
||||
|
|
|
|||
Loading…
Reference in New Issue