From eaa74d77d9747b4ff02aed87dcfe8d149edab5a0 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Tue, 10 Jan 2023 18:51:50 +0000 Subject: [PATCH 01/42] Metadata parsing Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 47 +++++++++++++---------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 512ff61a9..83494cd00 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -15,7 +15,6 @@ package eventhubs import ( "context" - "encoding/json" "errors" "fmt" "strconv" @@ -33,6 +32,7 @@ import ( azauth "github.com/dapr/components-contrib/internal/authentication/azure" "github.com/dapr/components-contrib/internal/utils" + "github.com/dapr/components-contrib/metadata" contribMetadata "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" @@ -47,18 +47,18 @@ const ( partitionKeyMetadataKey = "partitionKey" // errors. - hubManagerCreationErrorMsg = "error: creating eventHub manager client" - invalidConnectionStringErrorMsg = "error: connectionString is invalid" - missingConnectionStringNamespaceErrorMsg = "error: connectionString or eventHubNamespace is required" - missingStorageAccountNameErrorMsg = "error: storageAccountName is a required attribute for subscribe" - missingStorageAccountKeyErrorMsg = "error: storageAccountKey is required for subscribe when connectionString is provided" - missingStorageContainerNameErrorMsg = "error: storageContainerName is a required attribute for subscribe" - missingConsumerIDErrorMsg = "error: missing consumerID attribute for subscribe" - bothConnectionStringNamespaceErrorMsg = "error: both connectionString and eventHubNamespace are given, only one should be given" - missingResourceGroupNameMsg = "error: missing resourceGroupName attribute required for entityManagement" - missingSubscriptionIDMsg = "error: missing subscriptionID attribute required for entityManagement" - entityManagementConnectionStrMsg = "error: entity management support is not available with connectionString" - differentTopicConnectionStringErrorTmpl = "error: specified topic %s does not match the event hub name in the provided connectionString" + hubManagerCreationErrorMsg = "failed to create eventHub manager client" + invalidConnectionStringErrorMsg = "connectionString is invalid" + missingConnectionStringNamespaceErrorMsg = "connectionString or eventHubNamespace is required" + missingStorageAccountNameErrorMsg = "storageAccountName is a required attribute for subscribe" + missingStorageAccountKeyErrorMsg = "storageAccountKey is required for subscribe when connectionString is provided" + missingStorageContainerNameErrorMsg = "storageContainerName is a required attribute for subscribe" + missingConsumerIDErrorMsg = "missing consumerID attribute for subscribe" + bothConnectionStringNamespaceErrorMsg = "both connectionString and eventHubNamespace are given, only one should be given" + missingResourceGroupNameMsg = "missing resourceGroupName attribute required for entityManagement" + missingSubscriptionIDMsg = "missing subscriptionID attribute required for entityManagement" + entityManagementConnectionStrMsg = "entity management support is not available with connectionString" + differentTopicConnectionStringErrorTmpl = "specified topic %s does not match the Event Hub name in the provided connectionString" // Event Hubs SystemProperties names for metadata passthrough. sysPropSequenceNumber = "x-opt-sequence-number" @@ -85,8 +85,8 @@ const ( resourceGetTimeout time.Duration = 5 * time.Second // See https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-quotas for numbers. - maxMessageRetention = int32(90) - maxPartitionCount = int32(1024) + maxMessageRetention int32 = 90 + maxPartitionCount int32 = 1024 ) func subscribeHandler(ctx context.Context, topic string, getAllProperties bool, e *eventhub.Event, handler pubsub.Handler) error { @@ -177,23 +177,18 @@ func NewAzureEventHubs(logger logger.Logger) pubsub.PubSub { } func parseEventHubsMetadata(meta pubsub.Metadata) (*azureEventHubsMetadata, error) { - b, err := json.Marshal(meta.Properties) + var m azureEventHubsMetadata + err := metadata.DecodeMetadata(meta.Properties, &m) if err != nil { - return nil, err - } - - m := azureEventHubsMetadata{} - err = json.Unmarshal(b, &m) - if err != nil { - return nil, err + return nil, fmt.Errorf("failed to decode metada: %w", err) } if m.ConnectionString == "" && m.EventHubNamespace == "" { - return &m, errors.New(missingConnectionStringNamespaceErrorMsg) + return nil, errors.New(missingConnectionStringNamespaceErrorMsg) } if m.ConnectionString != "" && m.EventHubNamespace != "" { - return &m, errors.New(bothConnectionStringNamespaceErrorMsg) + return nil, errors.New(bothConnectionStringNamespaceErrorMsg) } return &m, nil @@ -209,7 +204,7 @@ func validateAndGetHubName(connectionString string) (string, error) { func (aeh *AzureEventHubs) ensureEventHub(ctx context.Context, hubName string) error { if aeh.hubManager == nil { - aeh.logger.Errorf("hubManager client not initialized properly.") + aeh.logger.Errorf("hubManager client not initialized properly") return fmt.Errorf("hubManager client not initialized properly") } entity, err := aeh.getHubEntity(ctx, hubName) From 9ef0bb9441576ee86b967fede08ea107f04c3e39 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Tue, 10 Jan 2023 20:41:50 +0000 Subject: [PATCH 02/42] mapstructure fixes Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 83494cd00..b426c8981 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -158,17 +158,17 @@ type AzureEventHubs struct { } type azureEventHubsMetadata struct { - ConnectionString string `json:"connectionString,omitempty"` - EventHubNamespace string `json:"eventHubNamespace,omitempty"` - ConsumerGroup string `json:"consumerID"` - StorageAccountName string `json:"storageAccountName,omitempty"` - StorageAccountKey string `json:"storageAccountKey,omitempty"` - StorageContainerName string `json:"storageContainerName,omitempty"` - EnableEntityManagement bool `json:"enableEntityManagement,omitempty,string"` - MessageRetentionInDays int32 `json:"messageRetentionInDays,omitempty,string"` - PartitionCount int32 `json:"partitionCount,omitempty,string"` - SubscriptionID string `json:"subscriptionID,omitempty"` - ResourceGroupName string `json:"resourceGroupName,omitempty"` + ConnectionString string `json:"connectionString,omitempty" mapstructure:"connectionString"` + EventHubNamespace string `json:"eventHubNamespace,omitempty" mapstructure:"eventHubNamespace"` + ConsumerGroup string `json:"consumerID" mapstructure:"consumerID"` + StorageAccountName string `json:"storageAccountName,omitempty" mapstructure:"storageAccountName"` + StorageAccountKey string `json:"storageAccountKey,omitempty" mapstructure:"storageAccountKey"` + StorageContainerName string `json:"storageContainerName,omitempty" mapstructure:"storageContainerName"` + EnableEntityManagement bool `json:"enableEntityManagement,omitempty,string" mapstructure:"enableEntityManagement"` + MessageRetentionInDays int32 `json:"messageRetentionInDays,omitempty,string" mapstructure:"messageRetentionInDays"` + PartitionCount int32 `json:"partitionCount,omitempty,string" mapstructure:"partitionCount"` + SubscriptionID string `json:"subscriptionID,omitempty" mapstructure:"subscriptionID"` + ResourceGroupName string `json:"resourceGroupName,omitempty" mapstructure:"resourceGroupName"` } // NewAzureEventHubs returns a new Azure Event hubs instance. From 0157bf0b72c0a9da979a5de71bdd19fc9c1f49dd Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Thu, 19 Jan 2023 00:12:10 +0000 Subject: [PATCH 03/42] WIP Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- go.mod | 3 +- go.sum | 9 +- internal/component/azure/servicebus/client.go | 2 +- pubsub/azure/eventhubs/conn/README.md | 8 + pubsub/azure/eventhubs/conn/conn.go | 92 ++ pubsub/azure/eventhubs/eventhubs.go | 804 ++++-------------- pubsub/azure/eventhubs/eventhubs.go.old | 716 ++++++++++++++++ pubsub/azure/eventhubs/metadata.go | 66 ++ .../bindings/azure/blobstorage/go.mod | 2 +- .../bindings/azure/blobstorage/go.sum | 4 +- .../bindings/azure/cosmosdb/go.mod | 2 +- .../bindings/azure/cosmosdb/go.sum | 4 +- .../bindings/azure/eventhubs/go.mod | 2 +- .../bindings/azure/eventhubs/go.sum | 4 +- .../bindings/azure/servicebusqueues/go.mod | 2 +- .../bindings/azure/servicebusqueues/go.sum | 4 +- .../bindings/azure/storagequeues/go.mod | 2 +- .../bindings/azure/storagequeues/go.sum | 4 +- .../pubsub/azure/eventhubs/go.mod | 10 +- .../pubsub/azure/eventhubs/go.sum | 24 +- .../pubsub/azure/servicebus/topics/go.mod | 2 +- .../pubsub/azure/servicebus/topics/go.sum | 4 +- .../secretstores/azure/keyvault/go.mod | 2 +- .../secretstores/azure/keyvault/go.sum | 4 +- .../state/azure/blobstorage/go.mod | 2 +- .../state/azure/blobstorage/go.sum | 4 +- .../certification/state/azure/cosmosdb/go.mod | 2 +- .../certification/state/azure/cosmosdb/go.sum | 4 +- .../state/azure/tablestorage/go.mod | 2 +- .../state/azure/tablestorage/go.sum | 4 +- 30 files changed, 1090 insertions(+), 704 deletions(-) create mode 100644 pubsub/azure/eventhubs/conn/README.md create mode 100644 pubsub/azure/eventhubs/conn/conn.go create mode 100644 pubsub/azure/eventhubs/eventhubs.go.old create mode 100644 pubsub/azure/eventhubs/metadata.go diff --git a/go.mod b/go.mod index 9bdfc94f9..5476a4910 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2 github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 + github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.3 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 github.com/Azure/azure-storage-blob-go v0.15.0 @@ -140,7 +141,7 @@ require ( github.com/99designs/keyring v1.2.1 // indirect github.com/AthenZ/athenz v1.10.39 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect diff --git a/go.sum b/go.sum index 0f659386e..7aefc3860 100644 --- a/go.sum +++ b/go.sum @@ -317,14 +317,17 @@ github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2/go.mod h1:Fy3bbChFm4c github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgDAUZ+I2bO8J7s8JxXmu9fhi2ss= github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1/go.mod h1:l3wvZkG9oW07GLBW5Cd0WwG5asOfJ8aqE8raUvNzLpk= github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 h1:82w8tzLcOwDP/Q35j/wEBPt0n0kVC3cjtPdD62G8UAk= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0/go.mod h1:S78i9yTr4o/nXlH76bKjGUye9Z2wSxO5Tz7GoDr4vfI= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 h1:Lg6BW0VPmCwcMlvOviL3ruHFO+H9tZNqscK0AeuFjGM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 h1:X/ePaAG8guM7j5WORT5eEIw7cGUxe9Ah1jEQJKLmmSo= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0/go.mod h1:5dog28UP3dd1BnCPFYvyHfsmA+Phmoezt+KWT5cZnyc= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.3 h1:27HVgIcvrKkRs5eJzHnyZdt71/EyB3clkiJQB0qyIa8= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.3/go.mod h1:Eo6WMP/iw9sp06+v8y030eReUwX6sULn5i3fxCDWPag= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 h1:BWeAAEzkCnL0ABVJqs+4mYudNch7oFGPtTlSmIWL8ms= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= @@ -1136,8 +1139,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= diff --git a/internal/component/azure/servicebus/client.go b/internal/component/azure/servicebus/client.go index caad6c1a5..cd2727fdf 100644 --- a/internal/component/azure/servicebus/client.go +++ b/internal/component/azure/servicebus/client.go @@ -66,7 +66,7 @@ func NewClient(metadata *Metadata, rawMetadata map[string]string) (*Client, erro } } } else { - settings, err := azauth.NewEnvironmentSettings(azauth.AzureServiceBusResourceName, rawMetadata) + settings, err := azauth.NewEnvironmentSettings("servicebus", rawMetadata) if err != nil { return nil, err } diff --git a/pubsub/azure/eventhubs/conn/README.md b/pubsub/azure/eventhubs/conn/README.md new file mode 100644 index 000000000..b131a6a0a --- /dev/null +++ b/pubsub/azure/eventhubs/conn/README.md @@ -0,0 +1,8 @@ +# conn + +This package is copied from the Azure SDK as it is an internal package in the upstream SDK. + +We can remove this once [Azure/azure-sdk-for-go#19840](https://github.com/Azure/azure-sdk-for-go/issues/19840) is fixed. + +- Source: https://github.com/Azure/azure-sdk-for-go/tree/sdk/messaging/azeventhubs/v0.4.0/sdk/messaging/azeventhubs/internal/conn +- License: Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. diff --git a/pubsub/azure/eventhubs/conn/conn.go b/pubsub/azure/eventhubs/conn/conn.go new file mode 100644 index 000000000..54f23fba2 --- /dev/null +++ b/pubsub/azure/eventhubs/conn/conn.go @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package conn + +import ( + "errors" + "fmt" + "net/url" + "strings" +) + +type ( + // ParsedConn is the structure of a parsed Service Bus or Event Hub connection string. + ParsedConn struct { + Namespace string + HubName string + + KeyName string + Key string + + SAS string + } +) + +// ParsedConnectionFromStr takes a string connection string from the Azure portal and returns the parsed representation. +// The method will return an error if the Endpoint, SharedAccessKeyName or SharedAccessKey is empty. +func ParsedConnectionFromStr(connStr string) (*ParsedConn, error) { + const ( + endpointKey = "Endpoint" + sharedAccessKeyNameKey = "SharedAccessKeyName" + sharedAccessKeyKey = "SharedAccessKey" + entityPathKey = "EntityPath" + sharedAccessSignatureKey = "SharedAccessSignature" + ) + + // We can parse two types of connection strings. + // 1. Connection strings generated from the portal (or elsewhere) that contain an embedded key and keyname. + // 2. A specially formatted connection string with an embedded SharedAccessSignature: + // Endpoint=sb://.servicebus.windows.net;SharedAccessSignature=SharedAccessSignature sr=.servicebus.windows.net&sig=&se=&skn=" + var namespace, hubName, keyName, secret, sas string + splits := strings.Split(connStr, ";") + + for _, split := range splits { + keyAndValue := strings.SplitN(split, "=", 2) + if len(keyAndValue) < 2 { + return nil, errors.New("failed parsing connection string due to unmatched key value separated by '='") + } + + // if a key value pair has `=` in the value, recombine them + key := keyAndValue[0] + value := strings.Join(keyAndValue[1:], "=") + switch { + case strings.EqualFold(endpointKey, key): + u, err := url.Parse(value) + if err != nil { + return nil, errors.New("failed parsing connection string due to an incorrectly formatted Endpoint value") + } + namespace = u.Host + case strings.EqualFold(sharedAccessKeyNameKey, key): + keyName = value + case strings.EqualFold(sharedAccessKeyKey, key): + secret = value + case strings.EqualFold(entityPathKey, key): + hubName = value + case strings.EqualFold(sharedAccessSignatureKey, key): + sas = value + } + } + + parsed := &ParsedConn{ + Namespace: namespace, + KeyName: keyName, + Key: secret, + HubName: hubName, + SAS: sas, + } + + if namespace == "" { + return parsed, fmt.Errorf("key %q must not be empty", endpointKey) + } + + if sas == "" && keyName == "" { + return parsed, fmt.Errorf("key %q must not be empty", sharedAccessKeyNameKey) + } + + if secret == "" && sas == "" { + return parsed, fmt.Errorf("key %q or %q cannot both be empty", sharedAccessKeyKey, sharedAccessSignatureKey) + } + + return parsed, nil +} diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index b426c8981..ded22b78c 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Dapr Authors +Copyright 2023 The Dapr Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -17,500 +17,72 @@ import ( "context" "errors" "fmt" - "strconv" - "strings" - "time" + "sync" - "github.com/Azure/azure-amqp-common-go/v4/aad" - "github.com/Azure/azure-amqp-common-go/v4/conn" - eventhub "github.com/Azure/azure-event-hubs-go/v3" - "github.com/Azure/azure-event-hubs-go/v3/eph" - "github.com/Azure/azure-event-hubs-go/v3/storage" - mgmt "github.com/Azure/azure-sdk-for-go/services/eventhub/mgmt/2017-04-01/eventhub" - "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" azauth "github.com/dapr/components-contrib/internal/authentication/azure" - "github.com/dapr/components-contrib/internal/utils" - "github.com/dapr/components-contrib/metadata" - contribMetadata "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" + "github.com/dapr/components-contrib/pubsub/azure/eventhubs/conn" "github.com/dapr/kit/logger" - "github.com/dapr/kit/retry" ) -const ( - - // connection string entity path key. - entityPathKey = "EntityPath" - // metadata partitionKey key. - partitionKeyMetadataKey = "partitionKey" - - // errors. - hubManagerCreationErrorMsg = "failed to create eventHub manager client" - invalidConnectionStringErrorMsg = "connectionString is invalid" - missingConnectionStringNamespaceErrorMsg = "connectionString or eventHubNamespace is required" - missingStorageAccountNameErrorMsg = "storageAccountName is a required attribute for subscribe" - missingStorageAccountKeyErrorMsg = "storageAccountKey is required for subscribe when connectionString is provided" - missingStorageContainerNameErrorMsg = "storageContainerName is a required attribute for subscribe" - missingConsumerIDErrorMsg = "missing consumerID attribute for subscribe" - bothConnectionStringNamespaceErrorMsg = "both connectionString and eventHubNamespace are given, only one should be given" - missingResourceGroupNameMsg = "missing resourceGroupName attribute required for entityManagement" - missingSubscriptionIDMsg = "missing subscriptionID attribute required for entityManagement" - entityManagementConnectionStrMsg = "entity management support is not available with connectionString" - differentTopicConnectionStringErrorTmpl = "specified topic %s does not match the Event Hub name in the provided connectionString" - - // Event Hubs SystemProperties names for metadata passthrough. - sysPropSequenceNumber = "x-opt-sequence-number" - sysPropEnqueuedTime = "x-opt-enqueued-time" - sysPropOffset = "x-opt-offset" - sysPropPartitionID = "x-opt-partition-id" - sysPropPartitionKey = "x-opt-partition-key" - sysPropIotHubDeviceConnectionID = "iothub-connection-device-id" - sysPropIotHubAuthGenerationID = "iothub-connection-auth-generation-id" - sysPropIotHubConnectionAuthMethod = "iothub-connection-auth-method" - sysPropIotHubConnectionModuleID = "iothub-connection-module-id" - sysPropIotHubEnqueuedTime = "iothub-enqueuedtime" - sysPropMessageID = "message-id" - - // Metadata field to ensure all Event Hub properties pass through - requireAllProperties = "requireAllProperties" - - defaultMessageRetentionInDays = 1 - defaultPartitionCount = 1 - - resourceCheckMaxRetry = 5 - resourceCheckMaxRetryInterval time.Duration = 5 * time.Minute - resourceCreationTimeout time.Duration = 15 * time.Second - resourceGetTimeout time.Duration = 5 * time.Second - - // See https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-quotas for numbers. - maxMessageRetention int32 = 90 - maxPartitionCount int32 = 1024 -) - -func subscribeHandler(ctx context.Context, topic string, getAllProperties bool, e *eventhub.Event, handler pubsub.Handler) error { - res := pubsub.NewMessage{Data: e.Data, Topic: topic, Metadata: map[string]string{}} - if e.SystemProperties.SequenceNumber != nil { - res.Metadata[sysPropSequenceNumber] = strconv.FormatInt(*e.SystemProperties.SequenceNumber, 10) - } - if e.SystemProperties.EnqueuedTime != nil { - res.Metadata[sysPropEnqueuedTime] = e.SystemProperties.EnqueuedTime.Format(time.RFC3339) - } - if e.SystemProperties.Offset != nil { - res.Metadata[sysPropOffset] = strconv.FormatInt(*e.SystemProperties.Offset, 10) - } - // According to azure-event-hubs-go docs, this will always be nil. - if e.SystemProperties.PartitionID != nil { - res.Metadata[sysPropPartitionID] = strconv.Itoa(int(*e.SystemProperties.PartitionID)) - } - // The following metadata properties are only present if event was generated by Azure IoT Hub. - if e.SystemProperties.PartitionKey != nil { - res.Metadata[sysPropPartitionKey] = *e.SystemProperties.PartitionKey - } - if e.SystemProperties.IoTHubDeviceConnectionID != nil { - res.Metadata[sysPropIotHubDeviceConnectionID] = *e.SystemProperties.IoTHubDeviceConnectionID - } - if e.SystemProperties.IoTHubAuthGenerationID != nil { - res.Metadata[sysPropIotHubAuthGenerationID] = *e.SystemProperties.IoTHubAuthGenerationID - } - if e.SystemProperties.IoTHubConnectionAuthMethod != nil { - res.Metadata[sysPropIotHubConnectionAuthMethod] = *e.SystemProperties.IoTHubConnectionAuthMethod - } - if e.SystemProperties.IoTHubConnectionModuleID != nil { - res.Metadata[sysPropIotHubConnectionModuleID] = *e.SystemProperties.IoTHubConnectionModuleID - } - if e.SystemProperties.IoTHubEnqueuedTime != nil { - res.Metadata[sysPropIotHubEnqueuedTime] = e.SystemProperties.IoTHubEnqueuedTime.Format(time.RFC3339) - } - // azure-event-hubs-go SDK pulls out the AMQP message-id property to the Event.ID property, map it from there. - if e.ID != "" { - res.Metadata[sysPropMessageID] = e.ID - } - // added properties if any ( includes application properties from iot-hub) - if getAllProperties { - if e.Properties != nil && len(e.Properties) > 0 { - for key, value := range e.Properties { - if str, ok := value.(string); ok { - res.Metadata[key] = str - } - } - } - } - - return handler(ctx, &res) -} - // AzureEventHubs allows sending/receiving Azure Event Hubs events. type AzureEventHubs struct { - metadata *azureEventHubsMetadata - logger logger.Logger - backOffConfig retry.Config - hubClients map[string]*eventhub.Hub - eventProcessors map[string]*eph.EventProcessorHost - hubManager *eventhub.HubManager - eventHubSettings azauth.EnvironmentSettings - managementSettings azauth.EnvironmentSettings - cgClient *mgmt.ConsumerGroupsClient - tokenProvider *aad.TokenProvider - storageCredential azblob.Credential - azureEnvironment *azure.Environment -} + metadata *azureEventHubsMetadata + logger logger.Logger -type azureEventHubsMetadata struct { - ConnectionString string `json:"connectionString,omitempty" mapstructure:"connectionString"` - EventHubNamespace string `json:"eventHubNamespace,omitempty" mapstructure:"eventHubNamespace"` - ConsumerGroup string `json:"consumerID" mapstructure:"consumerID"` - StorageAccountName string `json:"storageAccountName,omitempty" mapstructure:"storageAccountName"` - StorageAccountKey string `json:"storageAccountKey,omitempty" mapstructure:"storageAccountKey"` - StorageContainerName string `json:"storageContainerName,omitempty" mapstructure:"storageContainerName"` - EnableEntityManagement bool `json:"enableEntityManagement,omitempty,string" mapstructure:"enableEntityManagement"` - MessageRetentionInDays int32 `json:"messageRetentionInDays,omitempty,string" mapstructure:"messageRetentionInDays"` - PartitionCount int32 `json:"partitionCount,omitempty,string" mapstructure:"partitionCount"` - SubscriptionID string `json:"subscriptionID,omitempty" mapstructure:"subscriptionID"` - ResourceGroupName string `json:"resourceGroupName,omitempty" mapstructure:"resourceGroupName"` + clientsLock *sync.RWMutex + producerClients map[string]*azeventhubs.ProducerClient } // NewAzureEventHubs returns a new Azure Event hubs instance. func NewAzureEventHubs(logger logger.Logger) pubsub.PubSub { - return &AzureEventHubs{logger: logger} -} - -func parseEventHubsMetadata(meta pubsub.Metadata) (*azureEventHubsMetadata, error) { - var m azureEventHubsMetadata - err := metadata.DecodeMetadata(meta.Properties, &m) - if err != nil { - return nil, fmt.Errorf("failed to decode metada: %w", err) + return &AzureEventHubs{ + logger: logger, + clientsLock: &sync.RWMutex{}, + producerClients: make(map[string]*azeventhubs.ProducerClient, 1), } - - if m.ConnectionString == "" && m.EventHubNamespace == "" { - return nil, errors.New(missingConnectionStringNamespaceErrorMsg) - } - - if m.ConnectionString != "" && m.EventHubNamespace != "" { - return nil, errors.New(bothConnectionStringNamespaceErrorMsg) - } - - return &m, nil -} - -func validateAndGetHubName(connectionString string) (string, error) { - parsed, err := conn.ParsedConnectionFromStr(connectionString) - if err != nil { - return "", err - } - return parsed.HubName, nil -} - -func (aeh *AzureEventHubs) ensureEventHub(ctx context.Context, hubName string) error { - if aeh.hubManager == nil { - aeh.logger.Errorf("hubManager client not initialized properly") - return fmt.Errorf("hubManager client not initialized properly") - } - entity, err := aeh.getHubEntity(ctx, hubName) - if err != nil { - return err - } - if entity == nil { - if err := aeh.createHubEntity(ctx, hubName); err != nil { - return err - } - } - return nil -} - -func (aeh *AzureEventHubs) ensureSubscription(ctx context.Context, hubName string) error { - err := aeh.ensureEventHub(ctx, hubName) - if err != nil { - return err - } - _, err = aeh.getConsumerGroupsClient() - if err != nil { - return err - } - return aeh.createConsumerGroup(ctx, hubName) -} - -func (aeh *AzureEventHubs) getConsumerGroupsClient() (*mgmt.ConsumerGroupsClient, error) { - if aeh.cgClient != nil { - return aeh.cgClient, nil - } - client := mgmt.NewConsumerGroupsClientWithBaseURI(aeh.managementSettings.AzureEnvironment.ResourceManagerEndpoint, - aeh.metadata.SubscriptionID) - a, err := aeh.managementSettings.GetAuthorizer() - if err != nil { - return nil, err - } - client.Authorizer = a - aeh.cgClient = &client - return aeh.cgClient, nil -} - -func (aeh *AzureEventHubs) createConsumerGroup(parentCtx context.Context, hubName string) error { - create := false - backOffConfig := retry.DefaultConfig() - backOffConfig.Policy = retry.PolicyExponential - backOffConfig.MaxInterval = resourceCheckMaxRetryInterval - backOffConfig.MaxRetries = resourceCheckMaxRetry - - b := backOffConfig.NewBackOffWithContext(parentCtx) - - err := retry.NotifyRecover(func() error { - c, err := aeh.shouldCreateConsumerGroup(parentCtx, hubName) - if err == nil { - create = c - return nil - } - return err - }, b, func(_ error, _ time.Duration) { - aeh.logger.Errorf("Error checking for consumer group for EventHub : %s. Retrying...", hubName) - }, func() { - aeh.logger.Warnf("Successfully checked for consumer group in EventHub %s after it previously failed.", hubName) - }) - if err != nil { - return err - } - if create { - ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) - _, err = aeh.cgClient.CreateOrUpdate(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup, mgmt.ConsumerGroup{}) - cancel() - if err != nil { - return err - } - } - return nil -} - -func (aeh *AzureEventHubs) shouldCreateConsumerGroup(parentCtx context.Context, hubName string) (bool, error) { - ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) - g, err := aeh.cgClient.Get(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup) - cancel() - if err != nil { - if g.HasHTTPStatus(404) { - return true, nil - } - return false, err - } - if *g.Name == aeh.metadata.ConsumerGroup { - aeh.logger.Infof("consumer group %s exists for the requested topic/eventHub %s", aeh.metadata.ConsumerGroup, hubName) - } - return false, nil -} - -func (aeh *AzureEventHubs) getHubEntity(parentCtx context.Context, hubName string) (*eventhub.HubEntity, error) { - ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) - defer cancel() - return aeh.hubManager.Get(ctx, hubName) -} - -func (aeh *AzureEventHubs) createHubEntity(parentCtx context.Context, hubName string) error { - ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) - _, err := aeh.hubManager.Put(ctx, hubName, - eventhub.HubWithMessageRetentionInDays(aeh.metadata.MessageRetentionInDays), - eventhub.HubWithPartitionCount(aeh.metadata.PartitionCount)) - cancel() - if err != nil { - aeh.logger.Errorf("error creating event hub %s: %s", hubName, err) - return fmt.Errorf("error creating event hub %s: %s", hubName, err) - } - return nil -} - -func (aeh *AzureEventHubs) ensurePublisherClient(ctx context.Context, hubName string) error { - if aeh.metadata.EnableEntityManagement { - if err := aeh.ensureEventHub(ctx, hubName); err != nil { - return err - } - } - userAgent := "dapr-" + logger.DaprVersion - if aeh.metadata.ConnectionString != "" { - // Connect with connection string. - newConnectionString, err := aeh.constructConnectionStringFromTopic(hubName) - if err != nil { - return err - } - - hub, err := eventhub.NewHubFromConnectionString(newConnectionString, - eventhub.HubWithUserAgent(userAgent)) - if err != nil { - aeh.logger.Debugf("unable to connect to azure event hubs: %v", err) - return fmt.Errorf("unable to connect to azure event hubs: %v", err) - } - aeh.hubClients[hubName] = hub - } else { - if hubName == "" { - return errors.New("error: missing topic/hubName attribute with AAD connection") - } - - hub, err := eventhub.NewHub(aeh.metadata.EventHubNamespace, hubName, aeh.tokenProvider, eventhub.HubWithUserAgent(userAgent)) - if err != nil { - return fmt.Errorf("unable to connect to azure event hubs: %v", err) - } - aeh.hubClients[hubName] = hub - } - - return nil -} - -func (aeh *AzureEventHubs) ensureSubscriberClient(ctx context.Context, topic string, leaserCheckpointer *storage.LeaserCheckpointer) (*eph.EventProcessorHost, error) { - // connectionString given. - if aeh.metadata.ConnectionString != "" { - hubName, err := validateAndGetHubName(aeh.metadata.ConnectionString) - if err != nil { - return nil, fmt.Errorf("error parsing connection string %s", err) - } - if hubName != "" && hubName != topic { - return nil, fmt.Errorf("error: component cannot subscribe to requested topic %s with the given connectionString", topic) - } - if hubName == "" { - aeh.logger.Debugf("eventhub namespace connection string given. using topic as event hub entity path") - } - connectionString, err := aeh.constructConnectionStringFromTopic(topic) - if err != nil { - return nil, err - } - processor, err := eph.NewFromConnectionString( - ctx, connectionString, - leaserCheckpointer, - leaserCheckpointer, - eph.WithNoBanner(), - eph.WithConsumerGroup(aeh.metadata.ConsumerGroup), - ) - if err != nil { - return nil, err - } - aeh.logger.Debugf("processor initialized via connection string for topic %s", topic) - return processor, nil - } - // AAD connection. - processor, err := eph.New(ctx, - aeh.metadata.EventHubNamespace, - topic, - aeh.tokenProvider, - leaserCheckpointer, - leaserCheckpointer, - eph.WithNoBanner(), - eph.WithConsumerGroup(aeh.metadata.ConsumerGroup), - ) - if err != nil { - return nil, err - } - aeh.logger.Debugf("processor initialized via AAD for topic %s", topic) - - return processor, nil -} - -func (aeh *AzureEventHubs) createHubManager() error { - // Only AAD based authentication supported. - hubManager, err := eventhub.NewHubManagerFromAzureEnvironment(aeh.metadata.EventHubNamespace, aeh.tokenProvider, *aeh.eventHubSettings.AzureEnvironment) - if err != nil { - return fmt.Errorf("%s %s", hubManagerCreationErrorMsg, err) - } - aeh.hubManager = hubManager - - return nil -} - -func (aeh *AzureEventHubs) constructConnectionStringFromTopic(requestedTopic string) (string, error) { - hubName, err := validateAndGetHubName(aeh.metadata.ConnectionString) - if err != nil { - return "", err - } - if hubName != "" && hubName == requestedTopic { - return aeh.metadata.ConnectionString, nil - } else if hubName != "" { - return "", fmt.Errorf(differentTopicConnectionStringErrorTmpl, requestedTopic) - } - return aeh.metadata.ConnectionString + ";" + entityPathKey + "=" + requestedTopic, nil -} - -func (aeh *AzureEventHubs) validateEnitityManagementMetadata() error { - if aeh.metadata.MessageRetentionInDays <= 0 || aeh.metadata.MessageRetentionInDays > maxMessageRetention { - aeh.logger.Warnf("invalid/no message retention time period is given with entity management enabled, default value of %d is used", defaultMessageRetentionInDays) - aeh.metadata.MessageRetentionInDays = defaultMessageRetentionInDays - } - if aeh.metadata.PartitionCount <= 0 || aeh.metadata.PartitionCount > maxPartitionCount { - aeh.logger.Warnf("invalid/no partition count is given with entity management enabled, default value of %d is used", defaultPartitionCount) - aeh.metadata.PartitionCount = defaultPartitionCount - } - if aeh.metadata.ResourceGroupName == "" { - return errors.New(missingResourceGroupNameMsg) - } - if aeh.metadata.SubscriptionID == "" { - return errors.New(missingSubscriptionIDMsg) - } - return nil -} - -func (aeh *AzureEventHubs) validateSubscriptionAttributes() error { - m := *aeh.metadata - - if m.StorageAccountName == "" { - return errors.New(missingStorageAccountNameErrorMsg) - } - - if m.StorageAccountKey == "" && m.ConnectionString != "" { - return errors.New(missingStorageAccountKeyErrorMsg) - } - - if m.StorageContainerName == "" { - return errors.New(missingStorageContainerNameErrorMsg) - } - - if m.ConsumerGroup == "" { - return errors.New(missingConsumerIDErrorMsg) - } - return nil -} - -func (aeh *AzureEventHubs) getStoragePrefixString(topic string) string { - // empty string in the end of slice to have a suffix "-". - return strings.Join([]string{"dapr", topic, aeh.metadata.ConsumerGroup, ""}, "-") } // Init connects to Azure Event Hubs. func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { - m, parseErr := parseEventHubsMetadata(metadata) - if parseErr != nil { - return parseErr + m, err := parseEventHubsMetadata(metadata, aeh.logger) + if err != nil { + return err } - aeh.metadata = m - aeh.eventProcessors = map[string]*eph.EventProcessorHost{} - aeh.hubClients = map[string]*eventhub.Hub{} if aeh.metadata.ConnectionString != "" { - // Validate connectionString. - hubName, validateErr := validateAndGetHubName(aeh.metadata.ConnectionString) - if validateErr != nil { - return errors.New(invalidConnectionStringErrorMsg) - } - if hubName != "" { - aeh.logger.Infof("connectionString provided is specific to event hub %q. Publishing or subscribing to a topic that does not match this event hub will fail when attempted.", hubName) - } else { - aeh.logger.Infof("hubName not given in connectionString. connection established on first publish/subscribe") - aeh.logger.Debugf("req.Topic field in incoming requests honored") - } - if aeh.metadata.EnableEntityManagement { - // See https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-management-libraries - return errors.New(entityManagementConnectionStrMsg) - } - } else { - // Connect via AAD. - settings, sErr := azauth.NewEnvironmentSettings(azauth.AzureEventHubsResourceName, metadata.Properties) - if sErr != nil { - return sErr - } - aeh.eventHubSettings = settings - tokenProvider, err := aeh.eventHubSettings.GetAMQPTokenProvider() + // Connect using the connection string + var parsedConn *conn.ParsedConn + parsedConn, err = conn.ParsedConnectionFromStr(aeh.metadata.ConnectionString) if err != nil { - return fmt.Errorf("%s %s", hubManagerCreationErrorMsg, err) + return fmt.Errorf("connectionString is invalid: %w", err) } - aeh.tokenProvider = tokenProvider - aeh.logger.Info("connecting to Azure EventHubs via AAD. connection established on first publish/subscribe") - aeh.logger.Debugf("req.Topic field in incoming requests honored") - if aeh.metadata.EnableEntityManagement { + if parsedConn.HubName != "" { + aeh.logger.Infof(`The provided connection string is specific to the Event Hub ("entity path") '%s'; publishing or subscribing to a topic that does not match this Event Hub will fail when attempted`, parsedConn.HubName) + } else { + aeh.logger.Infof(`The provided connection string does not contain an Event Hub name ("entity path"); the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored`) + } + + aeh.metadata.hubName = parsedConn.HubName + } else { + // Connect via Azure AD + var env azauth.EnvironmentSettings + env, err = azauth.NewEnvironmentSettings("eventhubs", metadata.Properties) + if err != nil { + return fmt.Errorf("failed to initialize Azure AD credentials: %w", err) + } + aeh.metadata.aadTokenProvider, err = env.GetTokenCredential() + if err != nil { + return fmt.Errorf("failed to get Azure AD token credentials provider: %w", err) + } + + aeh.logger.Info("connecting to Azure Event Hub using Azure AD; the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored") + + /*if aeh.metadata.EnableEntityManagement { if err := aeh.validateEnitityManagementMetadata(); err != nil { return err } @@ -526,193 +98,137 @@ func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { return err } aeh.managementSettings = settings - } + }*/ } - // connect to the storage account. - if m.StorageAccountKey != "" { + // Connect to the Azure Storage account + /*if m.StorageAccountKey != "" { metadata.Properties["accountKey"] = m.StorageAccountKey } var storageCredsErr error aeh.storageCredential, aeh.azureEnvironment, storageCredsErr = azauth.GetAzureStorageBlobCredentials(aeh.logger, m.StorageAccountName, metadata.Properties) if storageCredsErr != nil { return fmt.Errorf("invalid storage credentials with error: %w", storageCredsErr) - } + }*/ - // Default retry configuration is used if no backOff properties are set. - if err := retry.DecodeConfigWithPrefix( - &aeh.backOffConfig, - metadata.Properties, - "backOff"); err != nil { - return err - } - - return nil -} - -// Publish sends data to Azure Event Hubs. -func (aeh *AzureEventHubs) Publish(ctx context.Context, req *pubsub.PublishRequest) error { - if _, ok := aeh.hubClients[req.Topic]; !ok { - if err := aeh.ensurePublisherClient(ctx, req.Topic); err != nil { - return fmt.Errorf("error on establishing hub connection: %s", err) - } - } - event := &eventhub.Event{Data: req.Data} - val, ok := req.Metadata[partitionKeyMetadataKey] - if ok { - event.PartitionKey = &val - } - err := aeh.hubClients[req.Topic].Send(ctx, event) - if err != nil { - return fmt.Errorf("error from publish: %s", err) - } - - return nil -} - -// BulkPublish sends data to Azure Event Hubs in bulk. -func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPublishRequest) (pubsub.BulkPublishResponse, error) { - if _, ok := aeh.hubClients[req.Topic]; !ok { - if err := aeh.ensurePublisherClient(ctx, req.Topic); err != nil { - err = fmt.Errorf("error on establishing hub connection: %s", err) - return pubsub.NewBulkPublishResponse(req.Entries, err), err - } - } - - // Create a slice of events to send. - events := make([]*eventhub.Event, len(req.Entries)) - for i, entry := range req.Entries { - events[i] = &eventhub.Event{Data: entry.Event} - if val, ok := entry.Metadata[partitionKeyMetadataKey]; ok { - events[i].PartitionKey = &val - } - } - - // Configure options for sending events. - opts := []eventhub.BatchOption{ - eventhub.BatchWithMaxSizeInBytes(utils.GetElemOrDefaultFromMap( - req.Metadata, contribMetadata.MaxBulkPubBytesKey, int(eventhub.DefaultMaxMessageSizeInBytes))), - } - - // Send events. - err := aeh.hubClients[req.Topic].SendBatch(ctx, eventhub.NewEventBatchIterator(events...), opts...) - if err != nil { - // Partial success is not supported by Azure Event Hubs. - // If an error occurs, all events are considered failed. - return pubsub.NewBulkPublishResponse(req.Entries, err), err - } - - return pubsub.BulkPublishResponse{}, nil -} - -// Subscribe receives data from Azure Event Hubs. -func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) error { - err := aeh.validateSubscriptionAttributes() - if err != nil { - return fmt.Errorf("error : error on subscribe %s", err) - } - if aeh.metadata.EnableEntityManagement { - if err = aeh.ensureSubscription(subscribeCtx, req.Topic); err != nil { - return err - } - } - - // Set topic name, consumerID prefix for partition checkpoint lease blob path. - // This is needed to support multiple consumers for the topic using the same storage container. - leaserPrefixOpt := storage.WithPrefixInBlobPath(aeh.getStoragePrefixString(req.Topic)) - leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(aeh.storageCredential, aeh.metadata.StorageAccountName, aeh.metadata.StorageContainerName, *aeh.azureEnvironment, leaserPrefixOpt) - if err != nil { - return err - } - - processor, err := aeh.ensureSubscriberClient(subscribeCtx, req.Topic, leaserCheckpointer) - if err != nil { - return err - } - - getAllProperties := false - if req.Metadata[requireAllProperties] != "" { - getAllProperties, err = strconv.ParseBool(req.Metadata[requireAllProperties]) - if err != nil { - aeh.logger.Errorf("invalid value for metadata : %s . Error: %v.", requireAllProperties, err) - } - } - - aeh.logger.Debugf("registering handler for topic %s", req.Topic) - _, err = processor.RegisterHandler(subscribeCtx, - func(_ context.Context, e *eventhub.Event) error { - // This component has built-in retries because Event Hubs doesn't support N/ACK for messages - b := aeh.backOffConfig.NewBackOffWithContext(subscribeCtx) - - retryerr := retry.NotifyRecover(func() error { - aeh.logger.Debugf("Processing EventHubs event %s/%s", req.Topic, e.ID) - - return subscribeHandler(subscribeCtx, req.Topic, getAllProperties, e, handler) - }, b, func(_ error, _ time.Duration) { - aeh.logger.Warnf("Error processing EventHubs event: %s/%s. Retrying...", req.Topic, e.ID) - }, func() { - aeh.logger.Warnf("Successfully processed EventHubs event after it previously failed: %s/%s", req.Topic, e.ID) - }) - if retryerr != nil { - aeh.logger.Errorf("Too many failed attempts at processing Eventhubs event: %s/%s. Error: %v.", req.Topic, e.ID, err) - } - return retryerr - }) - if err != nil { - return err - } - - err = processor.StartNonBlocking(subscribeCtx) - if err != nil { - return err - } - aeh.eventProcessors[req.Topic] = processor - - // Listen for context cancelation and stop processing messages - // This seems to be necessary because otherwise the processor isn't automatically closed on context cancelation - go func() { - <-subscribeCtx.Done() - stopCtx, stopCancel := context.WithTimeout(context.Background(), resourceGetTimeout) - stopErr := processor.Close(stopCtx) - stopCancel() - if stopErr != nil { - aeh.logger.Warnf("Error closing subscribe processor: %v", stopErr) - } - }() - - return nil -} - -func (aeh *AzureEventHubs) Close() (err error) { - flag := false - var ctx context.Context - var cancel context.CancelFunc - for topic, client := range aeh.hubClients { - ctx, cancel = context.WithTimeout(context.Background(), resourceGetTimeout) - err = client.Close(ctx) - cancel() - if err != nil { - flag = true - aeh.logger.Warnf("error closing publish client properly for topic/eventHub %s: %s", topic, err) - } - } - aeh.hubClients = map[string]*eventhub.Hub{} - for topic, client := range aeh.eventProcessors { - ctx, cancel = context.WithTimeout(context.Background(), resourceGetTimeout) - err = client.Close(ctx) - cancel() - if err != nil { - flag = true - aeh.logger.Warnf("error closing event processor host client properly for topic/eventHub %s: %s", topic, err) - } - } - aeh.eventProcessors = map[string]*eph.EventProcessorHost{} - if flag { - return errors.New("error closing event hub clients in a proper fashion") - } return nil } func (aeh *AzureEventHubs) Features() []pubsub.Feature { return nil } + +// Publish sends a message to Azure Event Hubs. +func (aeh *AzureEventHubs) Publish(ctx context.Context, req *pubsub.PublishRequest) error { + if req.Topic == "" { + return errors.New("parameter 'topic' is required") + } + + // Get the producer client + client, err := aeh.getProducerClientForTopic(ctx, req.Topic) + if err != nil { + return fmt.Errorf("error trying to establish a connection: %w", err) + } + + // Build the batch of messages + batchOpts := &azeventhubs.EventDataBatchOptions{} + if pk := req.Metadata["partitionKey"]; pk != "" { + batchOpts.PartitionKey = &pk + } + batch, err := client.NewEventDataBatch(ctx, batchOpts) + if err != nil { + return fmt.Errorf("error creating batch: %w", err) + } + err = batch.AddEventData(&azeventhubs.EventData{ + Body: req.Data, + ContentType: req.ContentType, + }, nil) + if err != nil { + return fmt.Errorf("error adding message to batch: %w", err) + } + + // Send the message + client.SendEventDataBatch(ctx, batch, nil) + if err != nil { + return fmt.Errorf("error publishing batch: %w", err) + } + + return nil +} + +func (aeh *AzureEventHubs) getProducerClientForTopic(ctx context.Context, topic string) (client *azeventhubs.ProducerClient, err error) { + // Check if we have the producer client in the cache + aeh.clientsLock.RLock() + client = aeh.producerClients[topic] + aeh.clientsLock.RUnlock() + if client != nil { + return client, nil + } + + // After acquiring a write lock, check again if the producer exists in the cache just in case another goroutine created it in the meanwhile + aeh.clientsLock.Lock() + defer aeh.clientsLock.Unlock() + + client = aeh.producerClients[topic] + if client != nil { + return client, nil + } + + // Start by creating a new entity if needed + if aeh.metadata.EnableEntityManagement { + // TODO: Create a new entity + } + + clientOpts := &azeventhubs.ProducerClientOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + } + + // Check if we're authenticating using a connection string + if aeh.metadata.ConnectionString != "" { + var connString string + connString, err = aeh.constructConnectionStringFromTopic(topic) + if err != nil { + return nil, err + } + client, err = azeventhubs.NewProducerClientFromConnectionString(connString, "", clientOpts) + if err != nil { + return nil, fmt.Errorf("unable to connect to Azure Event Hub using a connection string: %w", err) + } + } else { + client, err = azeventhubs.NewProducerClient(aeh.metadata.EventHubNamespace, topic, aeh.metadata.aadTokenProvider, nil) + if err != nil { + return nil, fmt.Errorf("unable to connect to Azure Event Hub using Azure AD: %w", err) + } + } + + // Store in the cache before returning + aeh.producerClients[topic] = client + return client, nil +} + +// BulkPublish sends data to Azure Event Hubs in bulk. +func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPublishRequest) (pubsub.BulkPublishResponse, error) { + return pubsub.BulkPublishResponse{}, nil +} + +// Subscribe receives data from Azure Event Hubs. +func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) error { + return nil +} + +func (aeh *AzureEventHubs) Close() (err error) { + return nil +} + +// Returns a connection string with the Event Hub name (entity path) set if not present +func (aeh *AzureEventHubs) constructConnectionStringFromTopic(topic string) (string, error) { + if aeh.metadata.hubName != "" { + if aeh.metadata.hubName != topic { + return "", fmt.Errorf("the requested topic '%s' does not match the Event Hub name in the connection string", topic) + } + return aeh.metadata.ConnectionString, nil + } + + connString := aeh.metadata.ConnectionString + ";EntityPath=" + topic + return connString, nil +} diff --git a/pubsub/azure/eventhubs/eventhubs.go.old b/pubsub/azure/eventhubs/eventhubs.go.old new file mode 100644 index 000000000..c7e67d4c0 --- /dev/null +++ b/pubsub/azure/eventhubs/eventhubs.go.old @@ -0,0 +1,716 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eventhubs + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/Azure/azure-amqp-common-go/v4/aad" + "github.com/Azure/azure-amqp-common-go/v4/conn" + eventhub "github.com/Azure/azure-event-hubs-go/v3" + "github.com/Azure/azure-event-hubs-go/v3/eph" + "github.com/Azure/azure-event-hubs-go/v3/storage" + mgmt "github.com/Azure/azure-sdk-for-go/services/eventhub/mgmt/2017-04-01/eventhub" + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/go-autorest/autorest/azure" + + azauth "github.com/dapr/components-contrib/internal/authentication/azure" + "github.com/dapr/components-contrib/internal/utils" + "github.com/dapr/components-contrib/metadata" + contribMetadata "github.com/dapr/components-contrib/metadata" + "github.com/dapr/components-contrib/pubsub" + "github.com/dapr/kit/logger" + "github.com/dapr/kit/retry" +) + +const ( + // connection string entity path key. + entityPathKey = "EntityPath" + // metadata partitionKey key. + partitionKeyMetadataKey = "partitionKey" + + // errors. + hubManagerCreationErrorMsg = "failed to create eventHub manager client" + invalidConnectionStringErrorMsg = "connectionString is invalid" + missingConnectionStringNamespaceErrorMsg = "connectionString or eventHubNamespace is required" + missingStorageAccountNameErrorMsg = "storageAccountName is a required attribute for subscribe" + missingStorageAccountKeyErrorMsg = "storageAccountKey is required for subscribe when connectionString is provided" + missingStorageContainerNameErrorMsg = "storageContainerName is a required attribute for subscribe" + missingConsumerIDErrorMsg = "missing consumerID attribute for subscribe" + bothConnectionStringNamespaceErrorMsg = "both connectionString and eventHubNamespace are given, only one should be given" + missingResourceGroupNameMsg = "missing resourceGroupName attribute required for entityManagement" + missingSubscriptionIDMsg = "missing subscriptionID attribute required for entityManagement" + entityManagementConnectionStrMsg = "entity management support is not available with connectionString" + differentTopicConnectionStringErrorTmpl = "specified topic %s does not match the Event Hub name in the provided connectionString" + + // Event Hubs SystemProperties names for metadata passthrough. + sysPropSequenceNumber = "x-opt-sequence-number" + sysPropEnqueuedTime = "x-opt-enqueued-time" + sysPropOffset = "x-opt-offset" + sysPropPartitionID = "x-opt-partition-id" + sysPropPartitionKey = "x-opt-partition-key" + sysPropIotHubDeviceConnectionID = "iothub-connection-device-id" + sysPropIotHubAuthGenerationID = "iothub-connection-auth-generation-id" + sysPropIotHubConnectionAuthMethod = "iothub-connection-auth-method" + sysPropIotHubConnectionModuleID = "iothub-connection-module-id" + sysPropIotHubEnqueuedTime = "iothub-enqueuedtime" + sysPropMessageID = "message-id" + + // Metadata field to ensure all Event Hub properties pass through + requireAllProperties = "requireAllProperties" + + defaultMessageRetentionInDays = 1 + defaultPartitionCount = 1 + + resourceCheckMaxRetry = 5 + resourceCheckMaxRetryInterval time.Duration = 5 * time.Minute + resourceCreationTimeout time.Duration = 15 * time.Second + resourceGetTimeout time.Duration = 5 * time.Second + + // See https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-quotas for numbers. + maxMessageRetention int32 = 90 + maxPartitionCount int32 = 1024 +) + +func subscribeHandler(ctx context.Context, topic string, getAllProperties bool, e *eventhub.Event, handler pubsub.Handler) error { + res := pubsub.NewMessage{Data: e.Data, Topic: topic, Metadata: map[string]string{}} + if e.SystemProperties.SequenceNumber != nil { + res.Metadata[sysPropSequenceNumber] = strconv.FormatInt(*e.SystemProperties.SequenceNumber, 10) + } + if e.SystemProperties.EnqueuedTime != nil { + res.Metadata[sysPropEnqueuedTime] = e.SystemProperties.EnqueuedTime.Format(time.RFC3339) + } + if e.SystemProperties.Offset != nil { + res.Metadata[sysPropOffset] = strconv.FormatInt(*e.SystemProperties.Offset, 10) + } + // According to azure-event-hubs-go docs, this will always be nil. + if e.SystemProperties.PartitionID != nil { + res.Metadata[sysPropPartitionID] = strconv.Itoa(int(*e.SystemProperties.PartitionID)) + } + // The following metadata properties are only present if event was generated by Azure IoT Hub. + if e.SystemProperties.PartitionKey != nil { + res.Metadata[sysPropPartitionKey] = *e.SystemProperties.PartitionKey + } + if e.SystemProperties.IoTHubDeviceConnectionID != nil { + res.Metadata[sysPropIotHubDeviceConnectionID] = *e.SystemProperties.IoTHubDeviceConnectionID + } + if e.SystemProperties.IoTHubAuthGenerationID != nil { + res.Metadata[sysPropIotHubAuthGenerationID] = *e.SystemProperties.IoTHubAuthGenerationID + } + if e.SystemProperties.IoTHubConnectionAuthMethod != nil { + res.Metadata[sysPropIotHubConnectionAuthMethod] = *e.SystemProperties.IoTHubConnectionAuthMethod + } + if e.SystemProperties.IoTHubConnectionModuleID != nil { + res.Metadata[sysPropIotHubConnectionModuleID] = *e.SystemProperties.IoTHubConnectionModuleID + } + if e.SystemProperties.IoTHubEnqueuedTime != nil { + res.Metadata[sysPropIotHubEnqueuedTime] = e.SystemProperties.IoTHubEnqueuedTime.Format(time.RFC3339) + } + // azure-event-hubs-go SDK pulls out the AMQP message-id property to the Event.ID property, map it from there. + if e.ID != "" { + res.Metadata[sysPropMessageID] = e.ID + } + // added properties if any ( includes application properties from iot-hub) + if getAllProperties { + if e.Properties != nil && len(e.Properties) > 0 { + for key, value := range e.Properties { + if str, ok := value.(string); ok { + res.Metadata[key] = str + } + } + } + } + + return handler(ctx, &res) +} + +// AzureEventHubs allows sending/receiving Azure Event Hubs events. +type AzureEventHubs struct { + metadata *azureEventHubsMetadata + logger logger.Logger + backOffConfig retry.Config + hubClients map[string]*eventhub.Hub + eventProcessors map[string]*eph.EventProcessorHost + hubManager *eventhub.HubManager + eventHubSettings azauth.EnvironmentSettings + managementSettings azauth.EnvironmentSettings + cgClient *mgmt.ConsumerGroupsClient + tokenProvider *aad.TokenProvider + storageCredential azblob.Credential + azureEnvironment *azure.Environment +} + +type azureEventHubsMetadata struct { + ConnectionString string `json:"connectionString,omitempty" mapstructure:"connectionString"` + EventHubNamespace string `json:"eventHubNamespace,omitempty" mapstructure:"eventHubNamespace"` + ConsumerGroup string `json:"consumerID" mapstructure:"consumerID"` + StorageAccountName string `json:"storageAccountName,omitempty" mapstructure:"storageAccountName"` + StorageAccountKey string `json:"storageAccountKey,omitempty" mapstructure:"storageAccountKey"` + StorageContainerName string `json:"storageContainerName,omitempty" mapstructure:"storageContainerName"` + EnableEntityManagement bool `json:"enableEntityManagement,omitempty,string" mapstructure:"enableEntityManagement"` + MessageRetentionInDays int32 `json:"messageRetentionInDays,omitempty,string" mapstructure:"messageRetentionInDays"` + PartitionCount int32 `json:"partitionCount,omitempty,string" mapstructure:"partitionCount"` + SubscriptionID string `json:"subscriptionID,omitempty" mapstructure:"subscriptionID"` + ResourceGroupName string `json:"resourceGroupName,omitempty" mapstructure:"resourceGroupName"` +} + +// NewAzureEventHubs returns a new Azure Event hubs instance. +func NewAzureEventHubs(logger logger.Logger) pubsub.PubSub { + return &AzureEventHubs{logger: logger} +} + +func parseEventHubsMetadata(meta pubsub.Metadata) (*azureEventHubsMetadata, error) { + var m azureEventHubsMetadata + err := metadata.DecodeMetadata(meta.Properties, &m) + if err != nil { + return nil, fmt.Errorf("failed to decode metada: %w", err) + } + + if m.ConnectionString == "" && m.EventHubNamespace == "" { + return nil, errors.New(missingConnectionStringNamespaceErrorMsg) + } + + if m.ConnectionString != "" && m.EventHubNamespace != "" { + return nil, errors.New(bothConnectionStringNamespaceErrorMsg) + } + + return &m, nil +} + +func validateAndGetHubName(connectionString string) (string, error) { + parsed, err := conn.ParsedConnectionFromStr(connectionString) + if err != nil { + return "", err + } + return parsed.HubName, nil +} + +func (aeh *AzureEventHubs) ensureEventHub(ctx context.Context, hubName string) error { + if aeh.hubManager == nil { + aeh.logger.Errorf("hubManager client not initialized properly") + return fmt.Errorf("hubManager client not initialized properly") + } + entity, err := aeh.getHubEntity(ctx, hubName) + if err != nil { + return err + } + if entity == nil { + if err := aeh.createHubEntity(ctx, hubName); err != nil { + return err + } + } + return nil +} + +func (aeh *AzureEventHubs) ensureSubscription(ctx context.Context, hubName string) error { + err := aeh.ensureEventHub(ctx, hubName) + if err != nil { + return err + } + _, err = aeh.getConsumerGroupsClient() + if err != nil { + return err + } + return aeh.createConsumerGroup(ctx, hubName) +} + +func (aeh *AzureEventHubs) getConsumerGroupsClient() (*mgmt.ConsumerGroupsClient, error) { + if aeh.cgClient != nil { + return aeh.cgClient, nil + } + client := mgmt.NewConsumerGroupsClientWithBaseURI(aeh.managementSettings.AzureEnvironment.ResourceManagerEndpoint, + aeh.metadata.SubscriptionID) + a, err := aeh.managementSettings.GetAuthorizer() + if err != nil { + return nil, err + } + client.Authorizer = a + aeh.cgClient = &client + return aeh.cgClient, nil +} + +func (aeh *AzureEventHubs) createConsumerGroup(parentCtx context.Context, hubName string) error { + create := false + backOffConfig := retry.DefaultConfig() + backOffConfig.Policy = retry.PolicyExponential + backOffConfig.MaxInterval = resourceCheckMaxRetryInterval + backOffConfig.MaxRetries = resourceCheckMaxRetry + + b := backOffConfig.NewBackOffWithContext(parentCtx) + + err := retry.NotifyRecover(func() error { + c, err := aeh.shouldCreateConsumerGroup(parentCtx, hubName) + if err == nil { + create = c + return nil + } + return err + }, b, func(_ error, _ time.Duration) { + aeh.logger.Errorf("Error checking for consumer group for EventHub : %s. Retrying...", hubName) + }, func() { + aeh.logger.Warnf("Successfully checked for consumer group in EventHub %s after it previously failed.", hubName) + }) + if err != nil { + return err + } + if create { + ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) + _, err = aeh.cgClient.CreateOrUpdate(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup, mgmt.ConsumerGroup{}) + cancel() + if err != nil { + return err + } + } + return nil +} + +func (aeh *AzureEventHubs) shouldCreateConsumerGroup(parentCtx context.Context, hubName string) (bool, error) { + ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) + g, err := aeh.cgClient.Get(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup) + cancel() + if err != nil { + if g.HasHTTPStatus(404) { + return true, nil + } + return false, err + } + if *g.Name == aeh.metadata.ConsumerGroup { + aeh.logger.Infof("consumer group %s exists for the requested topic/eventHub %s", aeh.metadata.ConsumerGroup, hubName) + } + return false, nil +} + +func (aeh *AzureEventHubs) getHubEntity(parentCtx context.Context, hubName string) (*eventhub.HubEntity, error) { + ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) + defer cancel() + return aeh.hubManager.Get(ctx, hubName) +} + +func (aeh *AzureEventHubs) createHubEntity(parentCtx context.Context, hubName string) error { + ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) + _, err := aeh.hubManager.Put(ctx, hubName, + eventhub.HubWithMessageRetentionInDays(aeh.metadata.MessageRetentionInDays), + eventhub.HubWithPartitionCount(aeh.metadata.PartitionCount)) + cancel() + if err != nil { + aeh.logger.Errorf("error creating event hub %s: %s", hubName, err) + return fmt.Errorf("error creating event hub %s: %s", hubName, err) + } + return nil +} + +func (aeh *AzureEventHubs) ensurePublisherClient(ctx context.Context, hubName string) error { + if aeh.metadata.EnableEntityManagement { + if err := aeh.ensureEventHub(ctx, hubName); err != nil { + return err + } + } + userAgent := "dapr-" + logger.DaprVersion + if aeh.metadata.ConnectionString != "" { + // Connect with connection string. + newConnectionString, err := aeh.constructConnectionStringFromTopic(hubName) + if err != nil { + return err + } + + hub, err := eventhub.NewHubFromConnectionString(newConnectionString, + eventhub.HubWithUserAgent(userAgent)) + if err != nil { + aeh.logger.Debugf("unable to connect to azure event hubs: %v", err) + return fmt.Errorf("unable to connect to azure event hubs: %v", err) + } + aeh.hubClients[hubName] = hub + } else { + if hubName == "" { + return errors.New("error: missing topic/hubName attribute with AAD connection") + } + + hub, err := eventhub.NewHub(aeh.metadata.EventHubNamespace, hubName, aeh.tokenProvider, eventhub.HubWithUserAgent(userAgent)) + if err != nil { + return fmt.Errorf("unable to connect to azure event hubs: %v", err) + } + aeh.hubClients[hubName] = hub + } + + return nil +} + +func (aeh *AzureEventHubs) ensureSubscriberClient(ctx context.Context, topic string, leaserCheckpointer *storage.LeaserCheckpointer) (*eph.EventProcessorHost, error) { + // connectionString given. + if aeh.metadata.ConnectionString != "" { + hubName, err := validateAndGetHubName(aeh.metadata.ConnectionString) + if err != nil { + return nil, fmt.Errorf("error parsing connection string %s", err) + } + if hubName != "" && hubName != topic { + return nil, fmt.Errorf("error: component cannot subscribe to requested topic %s with the given connectionString", topic) + } + if hubName == "" { + aeh.logger.Debugf("eventhub namespace connection string given. using topic as event hub entity path") + } + connectionString, err := aeh.constructConnectionStringFromTopic(topic) + if err != nil { + return nil, err + } + processor, err := eph.NewFromConnectionString( + ctx, connectionString, + leaserCheckpointer, + leaserCheckpointer, + eph.WithNoBanner(), + eph.WithConsumerGroup(aeh.metadata.ConsumerGroup), + ) + if err != nil { + return nil, err + } + aeh.logger.Debugf("processor initialized via connection string for topic %s", topic) + return processor, nil + } + // AAD connection. + processor, err := eph.New(ctx, + aeh.metadata.EventHubNamespace, + topic, + aeh.tokenProvider, + leaserCheckpointer, + leaserCheckpointer, + eph.WithNoBanner(), + eph.WithConsumerGroup(aeh.metadata.ConsumerGroup), + ) + if err != nil { + return nil, err + } + aeh.logger.Debugf("processor initialized via AAD for topic %s", topic) + + return processor, nil +} + +func (aeh *AzureEventHubs) createHubManager() error { + // Only AAD based authentication supported. + hubManager, err := eventhub.NewHubManagerFromAzureEnvironment(aeh.metadata.EventHubNamespace, aeh.tokenProvider, *aeh.eventHubSettings.AzureEnvironment) + if err != nil { + return fmt.Errorf("%s %s", hubManagerCreationErrorMsg, err) + } + aeh.hubManager = hubManager + + return nil +} + +func (aeh *AzureEventHubs) constructConnectionStringFromTopic(requestedTopic string) (string, error) { + hubName, err := validateAndGetHubName(aeh.metadata.ConnectionString) + if err != nil { + return "", err + } + if hubName != "" && hubName == requestedTopic { + return aeh.metadata.ConnectionString, nil + } else if hubName != "" { + return "", fmt.Errorf(differentTopicConnectionStringErrorTmpl, requestedTopic) + } + return aeh.metadata.ConnectionString + ";" + entityPathKey + "=" + requestedTopic, nil +} + +func (aeh *AzureEventHubs) validateEnitityManagementMetadata() error { + if aeh.metadata.MessageRetentionInDays <= 0 || aeh.metadata.MessageRetentionInDays > maxMessageRetention { + aeh.logger.Warnf("invalid/no message retention time period is given with entity management enabled, default value of %d is used", defaultMessageRetentionInDays) + aeh.metadata.MessageRetentionInDays = defaultMessageRetentionInDays + } + if aeh.metadata.PartitionCount <= 0 || aeh.metadata.PartitionCount > maxPartitionCount { + aeh.logger.Warnf("invalid/no partition count is given with entity management enabled, default value of %d is used", defaultPartitionCount) + aeh.metadata.PartitionCount = defaultPartitionCount + } + if aeh.metadata.ResourceGroupName == "" { + return errors.New(missingResourceGroupNameMsg) + } + if aeh.metadata.SubscriptionID == "" { + return errors.New(missingSubscriptionIDMsg) + } + return nil +} + +func (aeh *AzureEventHubs) validateSubscriptionAttributes() error { + m := *aeh.metadata + + if m.StorageAccountName == "" { + return errors.New("storageAccountName is a required attribute for subscribe") + } + + if m.StorageAccountKey == "" && m.ConnectionString != "" { + return errors.New("storageAccountKey is required for subscribe when connectionString is provided") + } + + if m.StorageContainerName == "" { + return errors.New("storageContainerName is a required attribute for subscribe") + } + + if m.ConsumerGroup == "" { + return errors.New("consumerID is a required attribute for subscribe") + } + return nil +} + +func (aeh *AzureEventHubs) getStoragePrefixString(topic string) string { + return fmt.Sprintf("dapr-%s-%s-", topic, aeh.metadata.ConsumerGroup) +} + +// Init connects to Azure Event Hubs. +func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { + m, parseErr := parseEventHubsMetadata(metadata) + if parseErr != nil { + return parseErr + } + + aeh.metadata = m + aeh.eventProcessors = map[string]*eph.EventProcessorHost{} + aeh.hubClients = map[string]*eventhub.Hub{} + + if aeh.metadata.ConnectionString != "" { + // Validate connectionString. + hubName, validateErr := validateAndGetHubName(aeh.metadata.ConnectionString) + if validateErr != nil { + return errors.New(invalidConnectionStringErrorMsg) + } + if hubName != "" { + aeh.logger.Infof("connectionString provided is specific to event hub %q. Publishing or subscribing to a topic that does not match this event hub will fail when attempted.", hubName) + } else { + aeh.logger.Infof("hubName not given in connectionString. connection established on first publish/subscribe") + aeh.logger.Debugf("req.Topic field in incoming requests honored") + } + if aeh.metadata.EnableEntityManagement { + // See https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-management-libraries + return errors.New(entityManagementConnectionStrMsg) + } + } else { + // Connect via AAD. + settings, sErr := azauth.NewEnvironmentSettings(azauth.AzureEventHubsResourceName, metadata.Properties) + if sErr != nil { + return sErr + } + aeh.eventHubSettings = settings + tokenProvider, err := aeh.eventHubSettings.GetAMQPTokenProvider() + if err != nil { + return fmt.Errorf("%s %s", hubManagerCreationErrorMsg, err) + } + aeh.tokenProvider = tokenProvider + aeh.logger.Info("connecting to Azure EventHubs via AAD. connection established on first publish/subscribe") + aeh.logger.Debugf("req.Topic field in incoming requests honored") + + if aeh.metadata.EnableEntityManagement { + if err := aeh.validateEnitityManagementMetadata(); err != nil { + return err + } + + // Create hubManager for eventHub management with AAD. + if managerCreateErr := aeh.createHubManager(); managerCreateErr != nil { + return managerCreateErr + } + + // Get Azure Management plane settings for creating consumer groups using event hubs management client. + settings, err := azauth.NewEnvironmentSettings("azure", metadata.Properties) + if err != nil { + return err + } + aeh.managementSettings = settings + } + } + + // connect to the storage account. + if m.StorageAccountKey != "" { + metadata.Properties["accountKey"] = m.StorageAccountKey + } + var storageCredsErr error + aeh.storageCredential, aeh.azureEnvironment, storageCredsErr = azauth.GetAzureStorageBlobCredentials(aeh.logger, m.StorageAccountName, metadata.Properties) + if storageCredsErr != nil { + return fmt.Errorf("invalid storage credentials with error: %w", storageCredsErr) + } + + // Default retry configuration is used if no backOff properties are set. + if err := retry.DecodeConfigWithPrefix( + &aeh.backOffConfig, + metadata.Properties, + "backOff"); err != nil { + return err + } + + return nil +} + +// Publish sends data to Azure Event Hubs. +func (aeh *AzureEventHubs) Publish(ctx context.Context, req *pubsub.PublishRequest) error { + if _, ok := aeh.hubClients[req.Topic]; !ok { + if err := aeh.ensurePublisherClient(ctx, req.Topic); err != nil { + return fmt.Errorf("error on establishing hub connection: %s", err) + } + } + event := &eventhub.Event{Data: req.Data} + val, ok := req.Metadata[partitionKeyMetadataKey] + if ok { + event.PartitionKey = &val + } + err := aeh.hubClients[req.Topic].Send(ctx, event) + if err != nil { + return fmt.Errorf("error from publish: %s", err) + } + + return nil +} + +// BulkPublish sends data to Azure Event Hubs in bulk. +func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPublishRequest) (pubsub.BulkPublishResponse, error) { + if _, ok := aeh.hubClients[req.Topic]; !ok { + if err := aeh.ensurePublisherClient(ctx, req.Topic); err != nil { + err = fmt.Errorf("error on establishing hub connection: %s", err) + return pubsub.NewBulkPublishResponse(req.Entries, err), err + } + } + + // Create a slice of events to send. + events := make([]*eventhub.Event, len(req.Entries)) + for i, entry := range req.Entries { + events[i] = &eventhub.Event{Data: entry.Event} + if val, ok := entry.Metadata[partitionKeyMetadataKey]; ok { + events[i].PartitionKey = &val + } + } + + // Configure options for sending events. + opts := []eventhub.BatchOption{ + eventhub.BatchWithMaxSizeInBytes(utils.GetElemOrDefaultFromMap( + req.Metadata, contribMetadata.MaxBulkPubBytesKey, int(eventhub.DefaultMaxMessageSizeInBytes))), + } + + // Send events. + err := aeh.hubClients[req.Topic].SendBatch(ctx, eventhub.NewEventBatchIterator(events...), opts...) + if err != nil { + // Partial success is not supported by Azure Event Hubs. + // If an error occurs, all events are considered failed. + return pubsub.NewBulkPublishResponse(req.Entries, err), err + } + + return pubsub.BulkPublishResponse{}, nil +} + +// Subscribe receives data from Azure Event Hubs. +func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) error { + err := aeh.validateSubscriptionAttributes() + if err != nil { + return fmt.Errorf("error : error on subscribe %s", err) + } + if aeh.metadata.EnableEntityManagement { + if err = aeh.ensureSubscription(subscribeCtx, req.Topic); err != nil { + return err + } + } + + // Set topic name, consumerID prefix for partition checkpoint lease blob path. + // This is needed to support multiple consumers for the topic using the same storage container. + leaserPrefixOpt := storage.WithPrefixInBlobPath(aeh.getStoragePrefixString(req.Topic)) + leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(aeh.storageCredential, aeh.metadata.StorageAccountName, aeh.metadata.StorageContainerName, *aeh.azureEnvironment, leaserPrefixOpt) + if err != nil { + return err + } + + processor, err := aeh.ensureSubscriberClient(subscribeCtx, req.Topic, leaserCheckpointer) + if err != nil { + return err + } + + getAllProperties := false + if req.Metadata[requireAllProperties] != "" { + getAllProperties, err = strconv.ParseBool(req.Metadata[requireAllProperties]) + if err != nil { + aeh.logger.Errorf("invalid value for metadata : %s . Error: %v.", requireAllProperties, err) + } + } + + aeh.logger.Debugf("registering handler for topic %s", req.Topic) + _, err = processor.RegisterHandler(subscribeCtx, + func(_ context.Context, e *eventhub.Event) error { + // This component has built-in retries because Event Hubs doesn't support N/ACK for messages + b := aeh.backOffConfig.NewBackOffWithContext(subscribeCtx) + + retryerr := retry.NotifyRecover(func() error { + aeh.logger.Debugf("Processing EventHubs event %s/%s", req.Topic, e.ID) + + return subscribeHandler(subscribeCtx, req.Topic, getAllProperties, e, handler) + }, b, func(_ error, _ time.Duration) { + aeh.logger.Warnf("Error processing EventHubs event: %s/%s. Retrying...", req.Topic, e.ID) + }, func() { + aeh.logger.Warnf("Successfully processed EventHubs event after it previously failed: %s/%s", req.Topic, e.ID) + }) + if retryerr != nil { + aeh.logger.Errorf("Too many failed attempts at processing Eventhubs event: %s/%s. Error: %v.", req.Topic, e.ID, err) + } + return retryerr + }) + if err != nil { + return err + } + + err = processor.StartNonBlocking(subscribeCtx) + if err != nil { + return err + } + aeh.eventProcessors[req.Topic] = processor + + // Listen for context cancelation and stop processing messages + // This seems to be necessary because otherwise the processor isn't automatically closed on context cancelation + go func() { + <-subscribeCtx.Done() + stopCtx, stopCancel := context.WithTimeout(context.Background(), resourceGetTimeout) + stopErr := processor.Close(stopCtx) + stopCancel() + if stopErr != nil { + aeh.logger.Warnf("Error closing subscribe processor: %v", stopErr) + } + }() + + return nil +} + +func (aeh *AzureEventHubs) Close() (err error) { + flag := false + var ctx context.Context + var cancel context.CancelFunc + for topic, client := range aeh.hubClients { + ctx, cancel = context.WithTimeout(context.Background(), resourceGetTimeout) + err = client.Close(ctx) + cancel() + if err != nil { + flag = true + aeh.logger.Warnf("error closing publish client properly for topic/eventHub %s: %s", topic, err) + } + } + aeh.hubClients = map[string]*eventhub.Hub{} + for topic, client := range aeh.eventProcessors { + ctx, cancel = context.WithTimeout(context.Background(), resourceGetTimeout) + err = client.Close(ctx) + cancel() + if err != nil { + flag = true + aeh.logger.Warnf("error closing event processor host client properly for topic/eventHub %s: %s", topic, err) + } + } + aeh.eventProcessors = map[string]*eph.EventProcessorHost{} + if flag { + return errors.New("error closing event hub clients in a proper fashion") + } + return nil +} + +func (aeh *AzureEventHubs) Features() []pubsub.Feature { + return nil +} diff --git a/pubsub/azure/eventhubs/metadata.go b/pubsub/azure/eventhubs/metadata.go new file mode 100644 index 000000000..c5fae300a --- /dev/null +++ b/pubsub/azure/eventhubs/metadata.go @@ -0,0 +1,66 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eventhubs + +import ( + "errors" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + + "github.com/dapr/components-contrib/metadata" + "github.com/dapr/components-contrib/pubsub" + "github.com/dapr/kit/logger" +) + +type azureEventHubsMetadata struct { + ConnectionString string `json:"connectionString" mapstructure:"connectionString"` + EventHubNamespace string `json:"eventHubNamespace" mapstructure:"eventHubNamespace"` + ConsumerGroup string `json:"consumerID" mapstructure:"consumerID"` + StorageAccountName string `json:"storageAccountName" mapstructure:"storageAccountName"` + StorageAccountKey string `json:"storageAccountKey" mapstructure:"storageAccountKey"` + StorageContainerName string `json:"storageContainerName" mapstructure:"storageContainerName"` + EnableEntityManagement bool `json:"enableEntityManagement,string" mapstructure:"enableEntityManagement"` + MessageRetentionInDays int32 `json:"messageRetentionInDays,string" mapstructure:"messageRetentionInDays"` + PartitionCount int32 `json:"partitionCount,string" mapstructure:"partitionCount"` + SubscriptionID string `json:"subscriptionID" mapstructure:"subscriptionID"` + ResourceGroupName string `json:"resourceGroupName" mapstructure:"resourceGroupName"` + + // Internal properties + hubName string + aadTokenProvider azcore.TokenCredential +} + +func parseEventHubsMetadata(meta pubsub.Metadata, log logger.Logger) (*azureEventHubsMetadata, error) { + var m azureEventHubsMetadata + err := metadata.DecodeMetadata(meta.Properties, &m) + if err != nil { + return nil, fmt.Errorf("failed to decode metada: %w", err) + } + + if m.ConnectionString == "" && m.EventHubNamespace == "" { + return nil, errors.New("one of connectionString or eventHubNamespace is required") + } + + if m.ConnectionString != "" && m.EventHubNamespace != "" { + return nil, errors.New("only one of connectionString or eventHubNamespace should be passed") + } + + if m.EnableEntityManagement && m.ConnectionString != "" { + m.EnableEntityManagement = false + log.Warn("entity management support is not available when connecting with a connection string") + } + + return &m, nil +} diff --git a/tests/certification/bindings/azure/blobstorage/go.mod b/tests/certification/bindings/azure/blobstorage/go.mod index ea078d1f9..0767de79a 100644 --- a/tests/certification/bindings/azure/blobstorage/go.mod +++ b/tests/certification/bindings/azure/blobstorage/go.mod @@ -19,7 +19,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/bindings/azure/blobstorage/go.sum b/tests/certification/bindings/azure/blobstorage/go.sum index 6a709d1c8..1d0796372 100644 --- a/tests/certification/bindings/azure/blobstorage/go.sum +++ b/tests/certification/bindings/azure/blobstorage/go.sum @@ -44,8 +44,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUX github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= diff --git a/tests/certification/bindings/azure/cosmosdb/go.mod b/tests/certification/bindings/azure/cosmosdb/go.mod index 3732c743f..5c5915411 100644 --- a/tests/certification/bindings/azure/cosmosdb/go.mod +++ b/tests/certification/bindings/azure/cosmosdb/go.mod @@ -22,7 +22,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/bindings/azure/cosmosdb/go.sum b/tests/certification/bindings/azure/cosmosdb/go.sum index 3f33d4efd..713077539 100644 --- a/tests/certification/bindings/azure/cosmosdb/go.sum +++ b/tests/certification/bindings/azure/cosmosdb/go.sum @@ -48,8 +48,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfR github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2 h1:yJegJqjhrMJ3Oe5s43jOTGL2AsE7pJyx+7Yqls/65tw= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2/go.mod h1:Fy3bbChFm4cZn6oIxYYqKB2FG3rBDxk3NZDLDJCHl+Q= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/bindings/azure/eventhubs/go.mod b/tests/certification/bindings/azure/eventhubs/go.mod index 9829c2bf3..59d9669ad 100644 --- a/tests/certification/bindings/azure/eventhubs/go.mod +++ b/tests/certification/bindings/azure/eventhubs/go.mod @@ -22,7 +22,7 @@ require ( github.com/Azure/azure-sdk-for-go v67.0.0+incompatible // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-amqp v0.18.0 // indirect diff --git a/tests/certification/bindings/azure/eventhubs/go.sum b/tests/certification/bindings/azure/eventhubs/go.sum index 09f4e0afe..153a891ab 100644 --- a/tests/certification/bindings/azure/eventhubs/go.sum +++ b/tests/certification/bindings/azure/eventhubs/go.sum @@ -48,8 +48,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUX github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/bindings/azure/servicebusqueues/go.mod b/tests/certification/bindings/azure/servicebusqueues/go.mod index affeae56d..fb0b95e5c 100644 --- a/tests/certification/bindings/azure/servicebusqueues/go.mod +++ b/tests/certification/bindings/azure/servicebusqueues/go.mod @@ -19,7 +19,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.3 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect diff --git a/tests/certification/bindings/azure/servicebusqueues/go.sum b/tests/certification/bindings/azure/servicebusqueues/go.sum index e3cc8c301..8d3bac195 100644 --- a/tests/certification/bindings/azure/servicebusqueues/go.sum +++ b/tests/certification/bindings/azure/servicebusqueues/go.sum @@ -44,8 +44,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUX github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.3 h1:27HVgIcvrKkRs5eJzHnyZdt71/EyB3clkiJQB0qyIa8= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.3/go.mod h1:Eo6WMP/iw9sp06+v8y030eReUwX6sULn5i3fxCDWPag= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= diff --git a/tests/certification/bindings/azure/storagequeues/go.mod b/tests/certification/bindings/azure/storagequeues/go.mod index d7bfddeb8..385e51817 100644 --- a/tests/certification/bindings/azure/storagequeues/go.mod +++ b/tests/certification/bindings/azure/storagequeues/go.mod @@ -19,7 +19,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/bindings/azure/storagequeues/go.sum b/tests/certification/bindings/azure/storagequeues/go.sum index 2862bd96b..6b52a2d22 100644 --- a/tests/certification/bindings/azure/storagequeues/go.sum +++ b/tests/certification/bindings/azure/storagequeues/go.sum @@ -44,8 +44,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUX github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/pubsub/azure/eventhubs/go.mod b/tests/certification/pubsub/azure/eventhubs/go.mod index f78b8ad38..64feba196 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.mod +++ b/tests/certification/pubsub/azure/eventhubs/go.mod @@ -17,23 +17,19 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect - github.com/Azure/azure-event-hubs-go/v3 v3.4.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go v67.0.0+incompatible // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect - github.com/Azure/go-amqp v0.18.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect - github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect @@ -47,7 +43,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/devigned/tab v0.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect @@ -87,7 +82,6 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/jhump/protoreflect v1.13.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.12 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/tests/certification/pubsub/azure/eventhubs/go.sum b/tests/certification/pubsub/azure/eventhubs/go.sum index d1bd36bf7..a0ebdc552 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.sum +++ b/tests/certification/pubsub/azure/eventhubs/go.sum @@ -37,25 +37,23 @@ github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a h1: github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a/go.mod h1:C0A1KeiVHs+trY6gUTPhhGammbrZ30ZfXRW/nuT7HLw= github.com/Azure/azure-amqp-common-go/v4 v4.0.0 h1:mV5O74KYmonn0ZXtwfMjGUtZ9Z+Hv7AIFVS1s03sRvo= github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8ONkEwRgWXqes3SUt1Ftrc= -github.com/Azure/azure-event-hubs-go/v3 v3.4.0 h1:LtH0nHkXivyV/GajOu5ZFC5sb/5KZ8j+9U8UsfHVTOo= -github.com/Azure/azure-event-hubs-go/v3 v3.4.0/go.mod h1:ODgt5C1/c73FToYj+mWojUJLXF877ALc6G4XnfRFlAY= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v67.0.0+incompatible h1:SVBwznSETB0Sipd0uyGJr7khLhJOFRUEUb+0JgkCvDo= -github.com/Azure/azure-sdk-for-go v67.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 h1:X/ePaAG8guM7j5WORT5eEIw7cGUxe9Ah1jEQJKLmmSo= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0/go.mod h1:5dog28UP3dd1BnCPFYvyHfsmA+Phmoezt+KWT5cZnyc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 h1:BWeAAEzkCnL0ABVJqs+4mYudNch7oFGPtTlSmIWL8ms= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= -github.com/Azure/go-amqp v0.18.0 h1:95bTiJq0oxjK1RUlt5T3HF/THj6jWTRZpSXMPSOJLz8= -github.com/Azure/go-amqp v0.18.0/go.mod h1:+bg0x3ce5+Q3ahCEXnCsGG3ETpDQe3MEVnOuT2ywPwc= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= @@ -74,10 +72,6 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= -github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= -github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= @@ -146,8 +140,6 @@ github.com/dapr/kit v0.0.3/go.mod h1:+vh2UIRT0KzFm5YJWfj7az4XVSdodys1OCz1WzNe1Eo github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devigned/tab v0.1.1 h1:3mD6Kb1mUOYeLpJvTVSDwSg5ZsfSxfvxGRTxRsJsITA= -github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= @@ -173,7 +165,6 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -353,10 +344,9 @@ github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7 github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.13.0 h1:zrrZqa7JAc2YGgPSzZZkmUXJ5G6NRPdxOg/9t7ISImA= github.com/jhump/protoreflect v1.13.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/tests/certification/pubsub/azure/servicebus/topics/go.mod b/tests/certification/pubsub/azure/servicebus/topics/go.mod index d622a6407..556b90d00 100644 --- a/tests/certification/pubsub/azure/servicebus/topics/go.mod +++ b/tests/certification/pubsub/azure/servicebus/topics/go.mod @@ -20,7 +20,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.3 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect diff --git a/tests/certification/pubsub/azure/servicebus/topics/go.sum b/tests/certification/pubsub/azure/servicebus/topics/go.sum index e3cc8c301..8d3bac195 100644 --- a/tests/certification/pubsub/azure/servicebus/topics/go.sum +++ b/tests/certification/pubsub/azure/servicebus/topics/go.sum @@ -44,8 +44,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUX github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.3 h1:27HVgIcvrKkRs5eJzHnyZdt71/EyB3clkiJQB0qyIa8= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.3/go.mod h1:Eo6WMP/iw9sp06+v8y030eReUwX6sULn5i3fxCDWPag= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= diff --git a/tests/certification/secretstores/azure/keyvault/go.mod b/tests/certification/secretstores/azure/keyvault/go.mod index 4d880dee4..3ec529696 100644 --- a/tests/certification/secretstores/azure/keyvault/go.mod +++ b/tests/certification/secretstores/azure/keyvault/go.mod @@ -18,7 +18,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect diff --git a/tests/certification/secretstores/azure/keyvault/go.sum b/tests/certification/secretstores/azure/keyvault/go.sum index 783f15db5..a66e57a38 100644 --- a/tests/certification/secretstores/azure/keyvault/go.sum +++ b/tests/certification/secretstores/azure/keyvault/go.sum @@ -44,8 +44,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUX github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 h1:82w8tzLcOwDP/Q35j/wEBPt0n0kVC3cjtPdD62G8UAk= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0/go.mod h1:S78i9yTr4o/nXlH76bKjGUye9Z2wSxO5Tz7GoDr4vfI= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 h1:Lg6BW0VPmCwcMlvOviL3ruHFO+H9tZNqscK0AeuFjGM= diff --git a/tests/certification/state/azure/blobstorage/go.mod b/tests/certification/state/azure/blobstorage/go.mod index 0ccb4c704..8a44850db 100644 --- a/tests/certification/state/azure/blobstorage/go.mod +++ b/tests/certification/state/azure/blobstorage/go.mod @@ -18,7 +18,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect diff --git a/tests/certification/state/azure/blobstorage/go.sum b/tests/certification/state/azure/blobstorage/go.sum index 6a709d1c8..1d0796372 100644 --- a/tests/certification/state/azure/blobstorage/go.sum +++ b/tests/certification/state/azure/blobstorage/go.sum @@ -44,8 +44,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUX github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= diff --git a/tests/certification/state/azure/cosmosdb/go.mod b/tests/certification/state/azure/cosmosdb/go.mod index b0c3e8b8f..c6b6ff121 100644 --- a/tests/certification/state/azure/cosmosdb/go.mod +++ b/tests/certification/state/azure/cosmosdb/go.mod @@ -20,7 +20,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/state/azure/cosmosdb/go.sum b/tests/certification/state/azure/cosmosdb/go.sum index 3c4989036..2b7c13c86 100644 --- a/tests/certification/state/azure/cosmosdb/go.sum +++ b/tests/certification/state/azure/cosmosdb/go.sum @@ -48,8 +48,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfR github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2 h1:yJegJqjhrMJ3Oe5s43jOTGL2AsE7pJyx+7Yqls/65tw= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2/go.mod h1:Fy3bbChFm4cZn6oIxYYqKB2FG3rBDxk3NZDLDJCHl+Q= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/state/azure/tablestorage/go.mod b/tests/certification/state/azure/tablestorage/go.mod index 1bf7b0432..2b9c5f9c9 100644 --- a/tests/certification/state/azure/tablestorage/go.mod +++ b/tests/certification/state/azure/tablestorage/go.mod @@ -19,7 +19,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/state/azure/tablestorage/go.sum b/tests/certification/state/azure/tablestorage/go.sum index 1a8c4f07e..48ebdac72 100644 --- a/tests/certification/state/azure/tablestorage/go.sum +++ b/tests/certification/state/azure/tablestorage/go.sum @@ -46,8 +46,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfR github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgDAUZ+I2bO8J7s8JxXmu9fhi2ss= github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1/go.mod h1:l3wvZkG9oW07GLBW5Cd0WwG5asOfJ8aqE8raUvNzLpk= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= From 48dd1dabef8c5446cd67731c435a96f3ffa11f5f Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Thu, 19 Jan 2023 01:27:48 +0000 Subject: [PATCH 04/42] Working on subscribing Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 197 +++++++++++++++++++++++----- pubsub/azure/eventhubs/metadata.go | 33 +++-- 2 files changed, 182 insertions(+), 48 deletions(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index ded22b78c..fb399f2d7 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -19,7 +19,9 @@ import ( "fmt" "sync" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints" azauth "github.com/dapr/components-contrib/internal/authentication/azure" "github.com/dapr/components-contrib/pubsub" @@ -32,16 +34,21 @@ type AzureEventHubs struct { metadata *azureEventHubsMetadata logger logger.Logger - clientsLock *sync.RWMutex - producerClients map[string]*azeventhubs.ProducerClient + producersLock *sync.RWMutex + producers map[string]*azeventhubs.ProducerClient + processorsLock *sync.RWMutex + processors map[string]*azeventhubs.Processor + checkpointStore azeventhubs.CheckpointStore } // NewAzureEventHubs returns a new Azure Event hubs instance. func NewAzureEventHubs(logger logger.Logger) pubsub.PubSub { return &AzureEventHubs{ - logger: logger, - clientsLock: &sync.RWMutex{}, - producerClients: make(map[string]*azeventhubs.ProducerClient, 1), + logger: logger, + producersLock: &sync.RWMutex{}, + producers: make(map[string]*azeventhubs.ProducerClient, 1), + processorsLock: &sync.RWMutex{}, + processors: make(map[string]*azeventhubs.Processor, 1), } } @@ -101,16 +108,6 @@ func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { }*/ } - // Connect to the Azure Storage account - /*if m.StorageAccountKey != "" { - metadata.Properties["accountKey"] = m.StorageAccountKey - } - var storageCredsErr error - aeh.storageCredential, aeh.azureEnvironment, storageCredsErr = azauth.GetAzureStorageBlobCredentials(aeh.logger, m.StorageAccountName, metadata.Properties) - if storageCredsErr != nil { - return fmt.Errorf("invalid storage credentials with error: %w", storageCredsErr) - }*/ - return nil } @@ -156,25 +153,54 @@ func (aeh *AzureEventHubs) Publish(ctx context.Context, req *pubsub.PublishReque return nil } +// BulkPublish sends data to Azure Event Hubs in bulk. +func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPublishRequest) (pubsub.BulkPublishResponse, error) { + return pubsub.BulkPublishResponse{}, nil +} + +// Subscribe receives data from Azure Event Hubs. +func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) error { + if aeh.metadata.ConsumerGroup == "" { + return errors.New("property consumerID is required to subscribe to an Event Hub topic") + } + if req.Topic == "" { + return errors.New("parameter 'topic' is required") + } + + // Get the processor client + processor, err := aeh.getProcessorForTopic(subscribeCtx, req.Topic) + if err != nil { + return fmt.Errorf("error trying to establish a connection: %w", err) + } + + return nil +} + +func (aeh *AzureEventHubs) Close() (err error) { + return nil +} + +// Returns a producer client for a given topic. +// If the client doesn't exist in the cache, it will create one. func (aeh *AzureEventHubs) getProducerClientForTopic(ctx context.Context, topic string) (client *azeventhubs.ProducerClient, err error) { // Check if we have the producer client in the cache - aeh.clientsLock.RLock() - client = aeh.producerClients[topic] - aeh.clientsLock.RUnlock() + aeh.producersLock.RLock() + client = aeh.producers[topic] + aeh.producersLock.RUnlock() if client != nil { return client, nil } // After acquiring a write lock, check again if the producer exists in the cache just in case another goroutine created it in the meanwhile - aeh.clientsLock.Lock() - defer aeh.clientsLock.Unlock() + aeh.producersLock.Lock() + defer aeh.producersLock.Unlock() - client = aeh.producerClients[topic] + client = aeh.producers[topic] if client != nil { return client, nil } - // Start by creating a new entity if needed + // Create a new entity if needed if aeh.metadata.EnableEntityManagement { // TODO: Create a new entity } @@ -195,32 +221,133 @@ func (aeh *AzureEventHubs) getProducerClientForTopic(ctx context.Context, topic return nil, fmt.Errorf("unable to connect to Azure Event Hub using a connection string: %w", err) } } else { - client, err = azeventhubs.NewProducerClient(aeh.metadata.EventHubNamespace, topic, aeh.metadata.aadTokenProvider, nil) + // Use Azure AD + client, err = azeventhubs.NewProducerClient(aeh.metadata.EventHubNamespace, topic, aeh.metadata.aadTokenProvider, clientOpts) if err != nil { return nil, fmt.Errorf("unable to connect to Azure Event Hub using Azure AD: %w", err) } } - // Store in the cache before returning - aeh.producerClients[topic] = client + // Store in the cache before returning it + aeh.producers[topic] = client return client, nil } -// BulkPublish sends data to Azure Event Hubs in bulk. -func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPublishRequest) (pubsub.BulkPublishResponse, error) { - return pubsub.BulkPublishResponse{}, nil +// Returns a processor for a given topic. +// If the processor doesn't exist in the cache, it will create one. +func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic string) (processor *azeventhubs.Processor, err error) { + // Check if we have the producer client in the cache + aeh.processorsLock.RLock() + processor = aeh.processors[topic] + aeh.processorsLock.RUnlock() + if processor != nil { + return processor, nil + } + + // After acquiring a write lock, check again if the producer exists in the cache just in case another goroutine created it in the meanwhile + aeh.processorsLock.Lock() + defer aeh.processorsLock.Unlock() + + processor = aeh.processors[topic] + if processor != nil { + return processor, nil + } + + // Init the checkpoint store + err = aeh.initCheckpointStore() + if err != nil { + return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err) + } + + // Create a new entity if needed + if aeh.metadata.EnableEntityManagement { + // TODO: Create a new entity + } + + // Create a consumer client + var consumerClient *azeventhubs.ConsumerClient + clientOpts := &azeventhubs.ConsumerClientOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + } + + // Check if we're authenticating using a connection string + if aeh.metadata.ConnectionString != "" { + var connString string + connString, err = aeh.constructConnectionStringFromTopic(topic) + if err != nil { + return nil, err + } + consumerClient, err = azeventhubs.NewConsumerClientFromConnectionString(connString, "", aeh.metadata.ConsumerGroup, clientOpts) + if err != nil { + return nil, fmt.Errorf("unable to connect to Azure Event Hub using a connection string: %w", err) + } + } else { + // Use Azure AD + consumerClient, err = azeventhubs.NewConsumerClient(aeh.metadata.EventHubNamespace, topic, aeh.metadata.ConsumerGroup, aeh.metadata.aadTokenProvider, clientOpts) + if err != nil { + return nil, fmt.Errorf("unable to connect to Azure Event Hub using Azure AD: %w", err) + } + } + + // Create the processor from the consumer client and checkpoint store + processor, err = azeventhubs.NewProcessor(consumerClient, aeh.checkpointStore, nil) + if err != nil { + return nil, fmt.Errorf("unable to create the processor: %w", err) + } + + // Store in the cache before returning it + aeh.processors[topic] = processor + return processor, nil } -// Subscribe receives data from Azure Event Hubs. -func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) error { +// Initializes the checkpoint store in the object. +// The initialization happens lazily before the first subscription is started +func (aeh *AzureEventHubs) initCheckpointStore() (err error) { + if aeh.metadata.StorageAccountName == "" { + return errors.New("property storageAccountName is required to subscribe to an Event Hub topic") + } + if aeh.metadata.StorageContainerName == "" { + return errors.New("property storageContainerName is required to subscribe to an Event Hub topic") + } + + checkpointStoreOpts := &checkpoints.BlobStoreOptions{ + ClientOptions: policy.ClientOptions{ + Telemetry: policy.TelemetryOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + }, + }, + } + var checkpointStore azeventhubs.CheckpointStore + if aeh.metadata.StorageConnectionString != "" { + // Authenticate with a connection string + checkpointStore, err = checkpoints.NewBlobStoreFromConnectionString(aeh.metadata.StorageConnectionString, aeh.metadata.StorageContainerName, checkpointStoreOpts) + if err != nil { + return fmt.Errorf("error creating checkpointer from connection string: %w", err) + } + } else if aeh.metadata.StorageAccountKey != "" { + // Authenticate with a shared key + // TODO: BLOCKED BY https://github.com/Azure/azure-sdk-for-go/issues/19842 + panic("unimplemented") + } else { + // Use Azure AD + // If Event Hub is authenticated using a connection string, we can't use Azure AD here + if aeh.metadata.ConnectionString != "" { + return errors.New("either one of storageConnectionString or storageAccountKey is required when subscribing to an Event Hub topic without using Azure AD") + } + // Use the global URL for Azure Storage + containerURL := fmt.Sprintf("https://%s.blob.%s/%s", aeh.metadata.StorageAccountName, "core.windows.net", aeh.metadata.StorageContainerName) + checkpointStore, err = checkpoints.NewBlobStore(containerURL, aeh.metadata.aadTokenProvider, checkpointStoreOpts) + if err != nil { + return fmt.Errorf("error creating checkpointer from Azure AD credentials: %w", err) + } + } + + // Store the checkpoint store in the object + aeh.checkpointStore = checkpointStore return nil } -func (aeh *AzureEventHubs) Close() (err error) { - return nil -} - -// Returns a connection string with the Event Hub name (entity path) set if not present +// Returns a connection string with the Event Hub name (entity path) set if not present. func (aeh *AzureEventHubs) constructConnectionStringFromTopic(topic string) (string, error) { if aeh.metadata.hubName != "" { if aeh.metadata.hubName != topic { diff --git a/pubsub/azure/eventhubs/metadata.go b/pubsub/azure/eventhubs/metadata.go index c5fae300a..fa3ded2c7 100644 --- a/pubsub/azure/eventhubs/metadata.go +++ b/pubsub/azure/eventhubs/metadata.go @@ -25,17 +25,18 @@ import ( ) type azureEventHubsMetadata struct { - ConnectionString string `json:"connectionString" mapstructure:"connectionString"` - EventHubNamespace string `json:"eventHubNamespace" mapstructure:"eventHubNamespace"` - ConsumerGroup string `json:"consumerID" mapstructure:"consumerID"` - StorageAccountName string `json:"storageAccountName" mapstructure:"storageAccountName"` - StorageAccountKey string `json:"storageAccountKey" mapstructure:"storageAccountKey"` - StorageContainerName string `json:"storageContainerName" mapstructure:"storageContainerName"` - EnableEntityManagement bool `json:"enableEntityManagement,string" mapstructure:"enableEntityManagement"` - MessageRetentionInDays int32 `json:"messageRetentionInDays,string" mapstructure:"messageRetentionInDays"` - PartitionCount int32 `json:"partitionCount,string" mapstructure:"partitionCount"` - SubscriptionID string `json:"subscriptionID" mapstructure:"subscriptionID"` - ResourceGroupName string `json:"resourceGroupName" mapstructure:"resourceGroupName"` + ConnectionString string `json:"connectionString" mapstructure:"connectionString"` + EventHubNamespace string `json:"eventHubNamespace" mapstructure:"eventHubNamespace"` + ConsumerGroup string `json:"consumerID" mapstructure:"consumerID"` + StorageConnectionString string `json:"storageConnectionString" mapstructure:"storageConnectionString"` + StorageAccountName string `json:"storageAccountName" mapstructure:"storageAccountName"` + StorageAccountKey string `json:"storageAccountKey" mapstructure:"storageAccountKey"` + StorageContainerName string `json:"storageContainerName" mapstructure:"storageContainerName"` + EnableEntityManagement bool `json:"enableEntityManagement,string" mapstructure:"enableEntityManagement"` + MessageRetentionInDays int32 `json:"messageRetentionInDays,string" mapstructure:"messageRetentionInDays"` + PartitionCount int32 `json:"partitionCount,string" mapstructure:"partitionCount"` + SubscriptionID string `json:"subscriptionID" mapstructure:"subscriptionID"` + ResourceGroupName string `json:"resourceGroupName" mapstructure:"resourceGroupName"` // Internal properties hubName string @@ -49,17 +50,23 @@ func parseEventHubsMetadata(meta pubsub.Metadata, log logger.Logger) (*azureEven return nil, fmt.Errorf("failed to decode metada: %w", err) } + // One and only one of connectionString and eventHubNamespace is required if m.ConnectionString == "" && m.EventHubNamespace == "" { return nil, errors.New("one of connectionString or eventHubNamespace is required") } - if m.ConnectionString != "" && m.EventHubNamespace != "" { return nil, errors.New("only one of connectionString or eventHubNamespace should be passed") } + // If both storageConnectionString and storageAccountKey are specified, show a warning because the connection string will take priority + if m.StorageConnectionString != "" && m.StorageAccountName != "" { + log.Warn("Property storageAccountKey is ignored when storageConnectionString is present") + } + + // Entity management is only possible when using Azure AD if m.EnableEntityManagement && m.ConnectionString != "" { m.EnableEntityManagement = false - log.Warn("entity management support is not available when connecting with a connection string") + log.Warn("Entity management support is not available when connecting with a connection string") } return &m, nil From 093669e8a8a9334c94de1a5503a666106086f4db Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Thu, 19 Jan 2023 20:12:00 +0000 Subject: [PATCH 05/42] WIP Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 211 +++++++++++++++++++++------- 1 file changed, 162 insertions(+), 49 deletions(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index fb399f2d7..20ea9841e 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "sync" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" @@ -29,26 +30,34 @@ import ( "github.com/dapr/kit/logger" ) +const ( + defaultMessageRetentionInDays = 1 + defaultPartitionCount = 1 + + resourceCheckMaxRetry = 5 + resourceCheckMaxRetryInterval = 5 * time.Minute + resourceCreationTimeout = 15 * time.Second + resourceGetTimeout = 5 * time.Second +) + // AzureEventHubs allows sending/receiving Azure Event Hubs events. type AzureEventHubs struct { metadata *azureEventHubsMetadata logger logger.Logger - producersLock *sync.RWMutex - producers map[string]*azeventhubs.ProducerClient - processorsLock *sync.RWMutex - processors map[string]*azeventhubs.Processor - checkpointStore azeventhubs.CheckpointStore + producersLock *sync.RWMutex + producers map[string]*azeventhubs.ProducerClient + checkpointStoreCache azeventhubs.CheckpointStore + checkpointStoreLock *sync.RWMutex } // NewAzureEventHubs returns a new Azure Event hubs instance. func NewAzureEventHubs(logger logger.Logger) pubsub.PubSub { return &AzureEventHubs{ - logger: logger, - producersLock: &sync.RWMutex{}, - producers: make(map[string]*azeventhubs.ProducerClient, 1), - processorsLock: &sync.RWMutex{}, - processors: make(map[string]*azeventhubs.Processor, 1), + logger: logger, + producersLock: &sync.RWMutex{}, + producers: make(map[string]*azeventhubs.ProducerClient, 1), + checkpointStoreLock: &sync.RWMutex{}, } } @@ -173,10 +182,106 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su return fmt.Errorf("error trying to establish a connection: %w", err) } + // Process all partition clients as they come in + go func() { + for { + // This will block until a new partition client is available + // It returns nil if processor.Run terminates or if the context is canceled + partitionClient := processor.NextPartitionClient(subscribeCtx) + if partitionClient == nil { + return + } + + // Once we get a partition client, process the events in a separate goroutine + go func() { + processErr := aeh.processEvents(subscribeCtx, partitionClient) + if processErr != nil { + aeh.logger.Errorf("Error processing events from partition client: %v", processErr) + } + }() + } + }() + + // Start the processor + go func() { + // This is a blocking call that runs until the context is canceled + err = processor.Run(subscribeCtx) + if err != nil { + aeh.logger.Errorf("Error from event processor: %v", err) + } + }() + return nil } +func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, partitionClient *azeventhubs.ProcessorPartitionClient) error { + // At the end of the method we need to do some cleanup and close the partition client + defer func() { + closeCtx, closeCancel := context.WithTimeout(context.Background(), resourceGetTimeout) + defer closeCancel() + closeErr := partitionClient.Close(closeCtx) + if closeErr != nil { + aeh.logger.Errorf("Error while closing partition client: %v", closeErr) + } + }() + + // Loop to receive messages + var ( + ctx context.Context + cancel context.CancelFunc + events []*azeventhubs.ReceivedEventData + err error + ) + for { + // TODO: Support setting a batch size + const batchSize = 1 + ctx, cancel = context.WithCancel(subscribeCtx) + events, err = partitionClient.ReceiveEvents(ctx, batchSize, nil) + cancel() + + // A DeadlineExceeded error means that the context timed out before we received the full batch of messages, and that's fine + if err != nil && !errors.Is(err, context.DeadlineExceeded) { + // If we get an error like ErrorCodeOwnershipLost, it means that the partition was rebalanced and we lost it + // We'll just stop this subscription and return + eventHubError := (*azeventhubs.Error)(nil) + if errors.As(err, &eventHubError) && eventHubError.Code == azeventhubs.ErrorCodeOwnershipLost { + aeh.logger.Debug("Partition client lost ownership; stopping") + return nil + } + + return fmt.Errorf("error receiving events: %w", err) + } + + // TODO: Comment out + aeh.logger.Debugf("Received batch with %d events", len(events)) + + if len(events) != 0 { + for _, event := range events { + // process the event in some way + fmt.Printf("Event received with body %v\n", event.Body) + } + + // Update the checkpoint with the last event received. If we lose ownership of this partition or have to restart the next owner will start from this point. + // This context inherits from the background one in case subscriptionCtx gets canceled + ctx, cancel = context.WithTimeout(context.Background(), resourceCreationTimeout) + err = partitionClient.UpdateCheckpoint(ctx, events[len(events)-1]) + cancel() + if err != nil { + return fmt.Errorf("failed to update checkpoint: %w", err) + } + } + } +} + func (aeh *AzureEventHubs) Close() (err error) { + // Acquire locks + aeh.checkpointStoreLock.Lock() + defer aeh.checkpointStoreLock.Unlock() + aeh.producersLock.Lock() + defer aeh.producersLock.Unlock() + + // Close all producers + return nil } @@ -233,28 +338,10 @@ func (aeh *AzureEventHubs) getProducerClientForTopic(ctx context.Context, topic return client, nil } -// Returns a processor for a given topic. -// If the processor doesn't exist in the cache, it will create one. -func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic string) (processor *azeventhubs.Processor, err error) { - // Check if we have the producer client in the cache - aeh.processorsLock.RLock() - processor = aeh.processors[topic] - aeh.processorsLock.RUnlock() - if processor != nil { - return processor, nil - } - - // After acquiring a write lock, check again if the producer exists in the cache just in case another goroutine created it in the meanwhile - aeh.processorsLock.Lock() - defer aeh.processorsLock.Unlock() - - processor = aeh.processors[topic] - if processor != nil { - return processor, nil - } - - // Init the checkpoint store - err = aeh.initCheckpointStore() +// Creates a processor for a given topic. +func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic string) (*azeventhubs.Processor, error) { + // Get the checkpoint store + checkpointStore, err := aeh.getCheckpointStore() if err != nil { return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err) } @@ -290,24 +377,49 @@ func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic strin } // Create the processor from the consumer client and checkpoint store - processor, err = azeventhubs.NewProcessor(consumerClient, aeh.checkpointStore, nil) + processor, err := azeventhubs.NewProcessor(consumerClient, checkpointStore, nil) if err != nil { return nil, fmt.Errorf("unable to create the processor: %w", err) } - // Store in the cache before returning it - aeh.processors[topic] = processor return processor, nil } -// Initializes the checkpoint store in the object. -// The initialization happens lazily before the first subscription is started -func (aeh *AzureEventHubs) initCheckpointStore() (err error) { +// Returns the checkpoint store from the object. If it doesn't exist, it lazily initializes it. +func (aeh *AzureEventHubs) getCheckpointStore() (azeventhubs.CheckpointStore, error) { + // Check if we have the checkpoint store + aeh.checkpointStoreLock.RLock() + if aeh.checkpointStoreCache != nil { + aeh.checkpointStoreLock.RUnlock() + return aeh.checkpointStoreCache, nil + } + aeh.checkpointStoreLock.RUnlock() + + // After acquiring a write lock, check again if the checkpoint store exists in case another goroutine created it in the meanwhile + aeh.checkpointStoreLock.Lock() + defer aeh.checkpointStoreLock.Unlock() + + if aeh.checkpointStoreCache != nil { + return aeh.checkpointStoreCache, nil + } + + // Init the checkpoint store and store it in the object + var err error + aeh.checkpointStoreCache, err = aeh.createCheckpointStore() + if err != nil { + return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err) + } + + return aeh.checkpointStoreCache, nil +} + +// Initializes a new checkpoint store +func (aeh *AzureEventHubs) createCheckpointStore() (checkpointStore azeventhubs.CheckpointStore, err error) { if aeh.metadata.StorageAccountName == "" { - return errors.New("property storageAccountName is required to subscribe to an Event Hub topic") + return nil, errors.New("property storageAccountName is required to subscribe to an Event Hub topic") } if aeh.metadata.StorageContainerName == "" { - return errors.New("property storageContainerName is required to subscribe to an Event Hub topic") + return nil, errors.New("property storageContainerName is required to subscribe to an Event Hub topic") } checkpointStoreOpts := &checkpoints.BlobStoreOptions{ @@ -317,34 +429,35 @@ func (aeh *AzureEventHubs) initCheckpointStore() (err error) { }, }, } - var checkpointStore azeventhubs.CheckpointStore if aeh.metadata.StorageConnectionString != "" { // Authenticate with a connection string checkpointStore, err = checkpoints.NewBlobStoreFromConnectionString(aeh.metadata.StorageConnectionString, aeh.metadata.StorageContainerName, checkpointStoreOpts) if err != nil { - return fmt.Errorf("error creating checkpointer from connection string: %w", err) + return nil, fmt.Errorf("error creating checkpointer from connection string: %w", err) } } else if aeh.metadata.StorageAccountKey != "" { // Authenticate with a shared key - // TODO: BLOCKED BY https://github.com/Azure/azure-sdk-for-go/issues/19842 - panic("unimplemented") + // TODO: This is a workaround in which we assemble a connection string until https://github.com/Azure/azure-sdk-for-go/issues/19842 is fixed + connString := fmt.Sprintf("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=core.windows.net", aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey) + checkpointStore, err = checkpoints.NewBlobStoreFromConnectionString(connString, aeh.metadata.StorageContainerName, checkpointStoreOpts) + if err != nil { + return nil, fmt.Errorf("error creating checkpointer from storage account credentials: %w", err) + } } else { // Use Azure AD // If Event Hub is authenticated using a connection string, we can't use Azure AD here if aeh.metadata.ConnectionString != "" { - return errors.New("either one of storageConnectionString or storageAccountKey is required when subscribing to an Event Hub topic without using Azure AD") + return nil, errors.New("either one of storageConnectionString or storageAccountKey is required when subscribing to an Event Hub topic without using Azure AD") } // Use the global URL for Azure Storage containerURL := fmt.Sprintf("https://%s.blob.%s/%s", aeh.metadata.StorageAccountName, "core.windows.net", aeh.metadata.StorageContainerName) checkpointStore, err = checkpoints.NewBlobStore(containerURL, aeh.metadata.aadTokenProvider, checkpointStoreOpts) if err != nil { - return fmt.Errorf("error creating checkpointer from Azure AD credentials: %w", err) + return nil, fmt.Errorf("error creating checkpointer from Azure AD credentials: %w", err) } } - // Store the checkpoint store in the object - aeh.checkpointStore = checkpointStore - return nil + return checkpointStore, nil } // Returns a connection string with the Event Hub name (entity path) set if not present. From 445f2e4940b7830e9b85001b784a0540f2f73ebc Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Thu, 19 Jan 2023 21:36:20 +0000 Subject: [PATCH 06/42] Receiving events Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 18 +++-- pubsub/azure/eventhubs/events.go | 108 ++++++++++++++++++++++++++++ pubsub/requests.go | 34 +++++++++ 3 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 pubsub/azure/eventhubs/events.go diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 20ea9841e..ccd05353b 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -25,6 +25,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints" azauth "github.com/dapr/components-contrib/internal/authentication/azure" + "github.com/dapr/components-contrib/internal/utils" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/components-contrib/pubsub/azure/eventhubs/conn" "github.com/dapr/kit/logger" @@ -168,7 +169,7 @@ func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPubl } // Subscribe receives data from Azure Event Hubs. -func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) error { +func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) (err error) { if aeh.metadata.ConsumerGroup == "" { return errors.New("property consumerID is required to subscribe to an Event Hub topic") } @@ -176,12 +177,17 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su return errors.New("parameter 'topic' is required") } + // Check if requireAllProperties is set and is truthy + getAllProperties := utils.IsTruthy(req.Metadata["requireAllProperties"]) + // Get the processor client processor, err := aeh.getProcessorForTopic(subscribeCtx, req.Topic) if err != nil { return fmt.Errorf("error trying to establish a connection: %w", err) } + eventHandler := subscribeHandler(subscribeCtx, req.Topic, getAllProperties, handler) + // Process all partition clients as they come in go func() { for { @@ -194,7 +200,7 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su // Once we get a partition client, process the events in a separate goroutine go func() { - processErr := aeh.processEvents(subscribeCtx, partitionClient) + processErr := aeh.processEvents(subscribeCtx, partitionClient, eventHandler) if processErr != nil { aeh.logger.Errorf("Error processing events from partition client: %v", processErr) } @@ -214,7 +220,7 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su return nil } -func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, partitionClient *azeventhubs.ProcessorPartitionClient) error { +func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, partitionClient *azeventhubs.ProcessorPartitionClient, eventHandler func(e *azeventhubs.ReceivedEventData) error) error { // At the end of the method we need to do some cleanup and close the partition client defer func() { closeCtx, closeCancel := context.WithTimeout(context.Background(), resourceGetTimeout) @@ -257,8 +263,10 @@ func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, partition if len(events) != 0 { for _, event := range events { - // process the event in some way - fmt.Printf("Event received with body %v\n", event.Body) + // TODO: Handle errors + // Maybe consider exiting this method (without updating the checkpoint) so another app will re-try it? + err := eventHandler(event) + fmt.Printf("Event processed - err: %v", err) } // Update the checkpoint with the last event received. If we lose ownership of this partition or have to restart the next owner will start from this point. diff --git a/pubsub/azure/eventhubs/events.go b/pubsub/azure/eventhubs/events.go new file mode 100644 index 000000000..4a8f7e11b --- /dev/null +++ b/pubsub/azure/eventhubs/events.go @@ -0,0 +1,108 @@ +package eventhubs + +import ( + "context" + "strconv" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" + "github.com/spf13/cast" + "golang.org/x/exp/maps" + + "github.com/dapr/components-contrib/pubsub" +) + +const ( + // Event Hubs SystemProperties names for metadata passthrough. + sysPropSequenceNumber = "x-opt-sequence-number" + sysPropEnqueuedTime = "x-opt-enqueued-time" + sysPropOffset = "x-opt-offset" + sysPropPartitionID = "x-opt-partition-id" + sysPropPartitionKey = "x-opt-partition-key" + sysPropIotHubDeviceConnectionID = "iothub-connection-device-id" + sysPropIotHubAuthGenerationID = "iothub-connection-auth-generation-id" + sysPropIotHubConnectionAuthMethod = "iothub-connection-auth-method" + sysPropIotHubConnectionModuleID = "iothub-connection-module-id" + sysPropIotHubEnqueuedTime = "iothub-enqueuedtime" + sysPropMessageID = "message-id" +) + +// Pool of map[string]string that we can use to optimize memory usage +var metadataPool = sync.Pool{ + New: func() any { + // Initial capacity is 5 which is the number of properties we normally find in a message + // The map can expand as needed, and when it's added back to the pool, we'll keep the larger size + m := make(map[string]string, 5) + return &m + }, +} + +func subscribeHandler(ctx context.Context, topic string, getAllProperties bool, handler pubsub.Handler) func(e *azeventhubs.ReceivedEventData) error { + return func(e *azeventhubs.ReceivedEventData) error { + md := metadataPool.Get().(*map[string]string) + maps.Clear(*md) + defer metadataPool.Put(md) + + res := pubsub.NewMessage{ + Data: e.Body, + Topic: topic, + Metadata: *md, + } + + res.Metadata[sysPropSequenceNumber] = strconv.FormatInt(e.SequenceNumber, 10) + if e.EnqueuedTime != nil { + res.Metadata[sysPropEnqueuedTime] = e.EnqueuedTime.Format(time.RFC3339) + } + if e.Offset != nil { + res.Metadata[sysPropOffset] = strconv.FormatInt(*e.Offset, 10) + } + if e.PartitionKey != nil { + res.Metadata[sysPropPartitionKey] = *e.PartitionKey + } + if e.MessageID != nil && *e.MessageID != "" { + res.Metadata[sysPropMessageID] = *e.MessageID + } + + // Iterate through the system properties looking for those coming from IoT Hub + for k, v := range e.SystemProperties { + switch k { + // The following metadata properties are only present if event was generated by Azure IoT Hub. + case sysPropIotHubDeviceConnectionID, + sysPropIotHubAuthGenerationID, + sysPropIotHubConnectionAuthMethod, + sysPropIotHubConnectionModuleID, + sysPropIotHubEnqueuedTime: + addPropertyToMetadata(k, v, res.Metadata) + default: + // nop + } + } + + // Added properties if any (includes application properties from Azure IoT Hub) + if getAllProperties && len(e.Properties) > 0 { + for k, v := range e.Properties { + addPropertyToMetadata(k, v, res.Metadata) + } + } + + return handler(ctx, &res) + } +} + +// Adds a property to the response metadata +func addPropertyToMetadata(key string, value any, md map[string]string) { + switch v := value.(type) { + case *time.Time: + if v != nil { + md[key] = v.Format(time.RFC3339) + } + case time.Time: + md[key] = v.Format(time.RFC3339) + default: + str, err := cast.ToStringE(value) + if err == nil { + md[key] = str + } + } +} diff --git a/pubsub/requests.go b/pubsub/requests.go index 490c29f8b..dcb32500d 100644 --- a/pubsub/requests.go +++ b/pubsub/requests.go @@ -13,6 +13,12 @@ limitations under the License. package pubsub +import ( + "encoding/json" + "fmt" + "strings" +) + // PublishRequest is the request to publish a message. type PublishRequest struct { Data []byte `json:"data"` @@ -45,6 +51,16 @@ type NewMessage struct { ContentType *string `json:"contentType,omitempty"` } +// String implements fmt.Stringer and it's useful for debugging. +func (m NewMessage) String() string { + ct := "(nil)" + if m.ContentType != nil { + ct = *m.ContentType + } + md, _ := json.Marshal(m.Metadata) + return fmt.Sprintf("[NewMessage] topic='%s' data='%s' content-type='%s' metadata=%s", m.Topic, string(m.Data), ct, md) +} + // BulkMessage represents bulk message arriving from a message bus instance. type BulkMessage struct { Entries []BulkMessageEntry `json:"entries"` @@ -52,6 +68,18 @@ type BulkMessage struct { Metadata map[string]string `json:"metadata"` } +// String implements fmt.Stringer and it's useful for debugging. +func (m BulkMessage) String() string { + md, _ := json.Marshal(m.Metadata) + b := strings.Builder{} + b.WriteString(fmt.Sprintf("[BulkMessage] topic='%s' metadata=%s entries=%d", m.Topic, md, len(m.Entries))) + for i, e := range m.Entries { + b.WriteString(fmt.Sprintf("\n%d: ", i)) + b.WriteString(e.String()) + } + return b.String() +} + // BulkMessageEntry represents a single message inside a bulk request. type BulkMessageEntry struct { EntryId string `json:"entryId"` //nolint:stylecheck @@ -60,6 +88,12 @@ type BulkMessageEntry struct { Metadata map[string]string `json:"metadata"` } +// String implements fmt.Stringer and it's useful for debugging. +func (m BulkMessageEntry) String() string { + md, _ := json.Marshal(m.Metadata) + return fmt.Sprintf("[BulkMessageEntry] entryId='%s' data='%s' content-type='%s' metadata=%s", m.EntryId, string(m.Event), m.ContentType, md) +} + // BulkSubscribeConfig represents the configuration for bulk subscribe. // It depends on specific componets to support these. type BulkSubscribeConfig struct { From 4c12f4053470d7bf3a733173cc8e51378774587e Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Thu, 19 Jan 2023 22:33:50 +0000 Subject: [PATCH 07/42] WIP Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 39 +++++++++++++++---- .../eventhubs/eventhubs_integration_test.go | 2 +- pubsub/azure/eventhubs/eventhubs_test.go | 2 +- pubsub/azure/eventhubs/events.go | 15 +++++++ 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index ccd05353b..78ba73d5a 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -23,6 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints" + "golang.org/x/exp/maps" azauth "github.com/dapr/components-contrib/internal/authentication/azure" "github.com/dapr/components-contrib/internal/utils" @@ -186,6 +187,7 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su return fmt.Errorf("error trying to establish a connection: %w", err) } + // Get the subscribe handler eventHandler := subscribeHandler(subscribeCtx, req.Topic, getAllProperties, handler) // Process all partition clients as they come in @@ -197,11 +199,13 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su if partitionClient == nil { return } + aeh.logger.Debugf("Received client for partition %s", partitionClient.PartitionID()) // Once we get a partition client, process the events in a separate goroutine go func() { - processErr := aeh.processEvents(subscribeCtx, partitionClient, eventHandler) - if processErr != nil { + processErr := aeh.processEvents(subscribeCtx, req.Topic, partitionClient, eventHandler) + // Do not log context.Canceled which happens at shutdown + if processErr != nil && !errors.Is(processErr, context.Canceled) { aeh.logger.Errorf("Error processing events from partition client: %v", processErr) } }() @@ -212,7 +216,8 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su go func() { // This is a blocking call that runs until the context is canceled err = processor.Run(subscribeCtx) - if err != nil { + // Do not log context.Canceled which happens at shutdown + if err != nil && !errors.Is(err, context.Canceled) { aeh.logger.Errorf("Error from event processor: %v", err) } }() @@ -220,7 +225,7 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su return nil } -func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, partitionClient *azeventhubs.ProcessorPartitionClient, eventHandler func(e *azeventhubs.ReceivedEventData) error) error { +func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, topic string, partitionClient *azeventhubs.ProcessorPartitionClient, eventHandler func(e *azeventhubs.ReceivedEventData) error) error { // At the end of the method we need to do some cleanup and close the partition client defer func() { closeCtx, closeCancel := context.WithTimeout(context.Background(), resourceGetTimeout) @@ -251,22 +256,21 @@ func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, partition // We'll just stop this subscription and return eventHubError := (*azeventhubs.Error)(nil) if errors.As(err, &eventHubError) && eventHubError.Code == azeventhubs.ErrorCodeOwnershipLost { - aeh.logger.Debug("Partition client lost ownership; stopping") + aeh.logger.Debug("Client lost ownership of partition %s for topic %s", partitionClient.PartitionID(), topic) return nil } return fmt.Errorf("error receiving events: %w", err) } - // TODO: Comment out - aeh.logger.Debugf("Received batch with %d events", len(events)) + aeh.logger.Debugf("Received batch with %d events on topic %s, partition %s", len(events), topic, partitionClient.PartitionID()) if len(events) != 0 { for _, event := range events { // TODO: Handle errors // Maybe consider exiting this method (without updating the checkpoint) so another app will re-try it? err := eventHandler(event) - fmt.Printf("Event processed - err: %v", err) + fmt.Printf("EVENT PROCESSED - err: %v\n", err) } // Update the checkpoint with the last event received. If we lose ownership of this partition or have to restart the next owner will start from this point. @@ -289,6 +293,25 @@ func (aeh *AzureEventHubs) Close() (err error) { defer aeh.producersLock.Unlock() // Close all producers + wg := sync.WaitGroup{} + for _, producer := range aeh.producers { + if producer == nil { + continue + } + wg.Add(1) + go func(producer *azeventhubs.ProducerClient) { + closeCtx, closeCancel := context.WithTimeout(context.Background(), resourceGetTimeout) + defer closeCancel() + producer.Close(closeCtx) + wg.Done() + }(producer) + } + wg.Wait() + maps.Clear(aeh.producers) + + // Remove the cached checkpoint store and metadata + aeh.checkpointStoreCache = nil + aeh.metadata = nil return nil } diff --git a/pubsub/azure/eventhubs/eventhubs_integration_test.go b/pubsub/azure/eventhubs/eventhubs_integration_test.go index 285063345..d9699365a 100644 --- a/pubsub/azure/eventhubs/eventhubs_integration_test.go +++ b/pubsub/azure/eventhubs/eventhubs_integration_test.go @@ -2,7 +2,7 @@ // +build integration_test /* -Copyright 2021 The Dapr Authors +Copyright 2023 The Dapr Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/pubsub/azure/eventhubs/eventhubs_test.go b/pubsub/azure/eventhubs/eventhubs_test.go index 1d46ca2ef..ceb61cc59 100644 --- a/pubsub/azure/eventhubs/eventhubs_test.go +++ b/pubsub/azure/eventhubs/eventhubs_test.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Dapr Authors +Copyright 2023 The Dapr Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/pubsub/azure/eventhubs/events.go b/pubsub/azure/eventhubs/events.go index 4a8f7e11b..248a6c4ee 100644 --- a/pubsub/azure/eventhubs/events.go +++ b/pubsub/azure/eventhubs/events.go @@ -1,3 +1,18 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package eventhubs import ( From c10173b29fdc97575624e27d049ff20d6ecbfb90 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 00:18:22 +0000 Subject: [PATCH 08/42] Added entity management Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- go.mod | 1 + go.sum | 1 + pubsub/azure/eventhubs/entity_management.go | 214 +++++++++++++++ pubsub/azure/eventhubs/eventhubs.go | 55 ++-- pubsub/azure/eventhubs/eventhubs.go.old | 1 - pubsub/azure/eventhubs/eventhubs_test.go | 287 ++++---------------- 6 files changed, 294 insertions(+), 265 deletions(-) create mode 100644 pubsub/azure/eventhubs/entity_management.go diff --git a/go.mod b/go.mod index ae610b033..962dd3044 100644 --- a/go.mod +++ b/go.mod @@ -144,6 +144,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect diff --git a/go.sum b/go.sum index c6b59da4f..dd58ee9ab 100644 --- a/go.sum +++ b/go.sum @@ -437,6 +437,7 @@ github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0/go.mod h1:5do github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4 h1:kaZamwZwmUqnECvnPkf1LBRBIFYYCy3E0gKHn/UFSD0= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4/go.mod h1:uDLwkzCJMvTrHsvtiVFeAp85hi3W77zvs61wrpc+6ho= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 h1:BWeAAEzkCnL0ABVJqs+4mYudNch7oFGPtTlSmIWL8ms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0/go.mod h1:Y3gnVwfaz8h6L1YHar+NfWORtBoVUSB5h4GlGkdeF7Q= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= diff --git a/pubsub/azure/eventhubs/entity_management.go b/pubsub/azure/eventhubs/entity_management.go new file mode 100644 index 000000000..85b8f99cc --- /dev/null +++ b/pubsub/azure/eventhubs/entity_management.go @@ -0,0 +1,214 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eventhubs + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub" + + azauth "github.com/dapr/components-contrib/internal/authentication/azure" + "github.com/dapr/kit/logger" + "github.com/dapr/kit/ptr" + "github.com/dapr/kit/retry" +) + +const ( + defaultMessageRetentionInDays = 1 + defaultPartitionCount = 1 + + resourceCheckMaxRetry = 5 + resourceCheckMaxRetryInterval = 5 * time.Minute + resourceCreationTimeout = 15 * time.Second + resourceGetTimeout = 5 * time.Second + + // See https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-quotas for numbers. + maxMessageRetention = int32(90) + maxPartitionCount = int32(1024) +) + +// Intializes the entity management capabilities. This method is invoked by Init. +func (aeh *AzureEventHubs) initEntityManagement(md map[string]string) error { + // Validate the metadata + err := aeh.validateEnitityManagementMetadata() + if err != nil { + return err + } + + // Get Azure Management plane credentials object + settings, err := azauth.NewEnvironmentSettings("azure", md) + if err != nil { + return err + } + creds, err := settings.GetTokenCredential() + if err != nil { + return fmt.Errorf("failed to obtain Azure AD management credentials: %w", err) + } + aeh.managementCreds = creds + return nil +} + +func (aeh *AzureEventHubs) validateEnitityManagementMetadata() error { + if aeh.metadata.MessageRetentionInDays <= 0 || aeh.metadata.MessageRetentionInDays > maxMessageRetention { + aeh.logger.Warnf("Property messageRetentionInDays for entity management has an empty or invalid value; using the default value %d", defaultMessageRetentionInDays) + aeh.metadata.MessageRetentionInDays = defaultMessageRetentionInDays + } + if aeh.metadata.PartitionCount <= 0 || aeh.metadata.PartitionCount > maxPartitionCount { + aeh.logger.Warnf("Property partitionCount for entity management has an empty or invalid value; using the default value %d", defaultPartitionCount) + aeh.metadata.PartitionCount = defaultPartitionCount + } + if aeh.metadata.ResourceGroupName == "" { + return errors.New("property resourceGroupName is required for entity management") + } + if aeh.metadata.SubscriptionID == "" { + return errors.New("property subscriptionID is required for entity management") + } + return nil +} + +// Ensures that the Event Hub entity exists. +// This is used during the creation of both producers and consumers. +func (aeh *AzureEventHubs) ensureEventHubEntity(parentCtx context.Context, topic string) error { + client, err := armeventhub.NewEventHubsClient(aeh.metadata.SubscriptionID, aeh.managementCreds, &arm.ClientOptions{ + ClientOptions: policy.ClientOptions{ + Telemetry: policy.TelemetryOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + }, + }, + }) + if err != nil { + return fmt.Errorf("failed to create Event Hubs ARM client: %w", err) + } + + aeh.logger.Debugf("Checking if entity %s exists on Event Hub namespace %s", topic, aeh.metadata.EventHubNamespace) + + // Check if the entity exists + ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) + defer cancel() + _, err = client.Get(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, topic, nil) + if err == nil { + // If there's no error, the entity already exists, so all good + aeh.logger.Debugf("Entity %s already exists on Event Hub namespace %s", topic, aeh.metadata.EventHubNamespace) + return nil + } + + // Check if the error is NotFound or something else + resErr := &azcore.ResponseError{} + if !errors.As(err, &resErr) || resErr.StatusCode != http.StatusNotFound { + // We have another error, just return it + return fmt.Errorf("failed to retrieve Event Hub entity %s: %w", topic, err) + } + + // Create the entity + aeh.logger.Infof("Will create entity %s on Event Hub namespace %s", topic, aeh.metadata.EventHubNamespace) + params := armeventhub.Eventhub{ + Properties: &armeventhub.Properties{ + MessageRetentionInDays: ptr.Of(int64(aeh.metadata.MessageRetentionInDays)), + PartitionCount: ptr.Of(int64(aeh.metadata.PartitionCount)), + }, + } + ctx, cancel = context.WithTimeout(parentCtx, resourceCreationTimeout) + defer cancel() + _, err = client.CreateOrUpdate(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, topic, params, nil) + if err != nil { + return fmt.Errorf("failed to create Event Hub entity %s: %w", topic, err) + } + + aeh.logger.Infof("Entity %s created on Event Hub namespace %s", topic, aeh.metadata.EventHubNamespace) + return nil +} + +// Ensures that the subscription (consumer group) exists. +// This is used during the creation of consumers only. +func (aeh *AzureEventHubs) ensureSubscription(parentCtx context.Context, hubName string) error { + client, err := armeventhub.NewConsumerGroupsClient(aeh.metadata.SubscriptionID, aeh.managementCreds, &arm.ClientOptions{ + ClientOptions: policy.ClientOptions{ + Telemetry: policy.TelemetryOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + }, + }, + }) + if err != nil { + return fmt.Errorf("failed to create consumer group ARM client: %w", err) + } + + aeh.logger.Debugf("Checking if consumer group %s exists in entity %s", aeh.metadata.ConsumerGroup, hubName) + + // Check if the consumer group exists + // We need to use a retry logic here + backOffConfig := retry.DefaultConfig() + backOffConfig.Policy = retry.PolicyExponential + backOffConfig.MaxInterval = resourceCheckMaxRetryInterval + backOffConfig.MaxRetries = resourceCheckMaxRetry + b := backOffConfig.NewBackOffWithContext(parentCtx) + create, err := retry.NotifyRecoverWithData(func() (bool, error) { + c, err := aeh.shouldCreateConsumerGroup(parentCtx, client, hubName) + if err != nil { + return false, err + } + return c, nil + }, b, func(_ error, _ time.Duration) { + aeh.logger.Errorf("Error checking for consumer group for Event Hub: %s. Retrying…", hubName) + }, func() { + aeh.logger.Warnf("Successfully checked for consumer group in Event Hub %s after it previously failed", hubName) + }) + if err != nil { + return err + } + if !create { + // Already exists + aeh.logger.Debugf("Consumer group %s already exists in entity %s", aeh.metadata.ConsumerGroup, hubName) + return nil + } + + // Need to create the consumer group + aeh.logger.Infof("Will create consumer group %s exists in entity %s", aeh.metadata.ConsumerGroup, hubName) + + ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) + defer cancel() + _, err = client.CreateOrUpdate(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup, armeventhub.ConsumerGroup{}, nil) + if err != nil { + return fmt.Errorf("failed to create consumer group %s: %w", aeh.metadata.ConsumerGroup, err) + } + + aeh.logger.Infof("Consumer group %s created in entity %s", aeh.metadata.ConsumerGroup, hubName) + return nil +} + +func (aeh *AzureEventHubs) shouldCreateConsumerGroup(parentCtx context.Context, client *armeventhub.ConsumerGroupsClient, hubName string) (bool, error) { + ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) + defer cancel() + _, err := client.Get(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup, nil) + if err == nil { + // If there's no error, the consumer group already exists, so all good + return true, nil + } + + // Check if the error is NotFound or something else + resErr := &azcore.ResponseError{} + if !errors.As(err, &resErr) || resErr.StatusCode != http.StatusNotFound { + // We have another error, just return it + return false, err + } + + // Consumer group doesn't exist + return false, nil +} diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 78ba73d5a..edf4f6dde 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -18,8 +18,8 @@ import ( "errors" "fmt" "sync" - "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints" @@ -32,16 +32,6 @@ import ( "github.com/dapr/kit/logger" ) -const ( - defaultMessageRetentionInDays = 1 - defaultPartitionCount = 1 - - resourceCheckMaxRetry = 5 - resourceCheckMaxRetryInterval = 5 * time.Minute - resourceCreationTimeout = 15 * time.Second - resourceGetTimeout = 5 * time.Second -) - // AzureEventHubs allows sending/receiving Azure Event Hubs events. type AzureEventHubs struct { metadata *azureEventHubsMetadata @@ -51,6 +41,8 @@ type AzureEventHubs struct { producers map[string]*azeventhubs.ProducerClient checkpointStoreCache azeventhubs.CheckpointStore checkpointStoreLock *sync.RWMutex + + managementCreds azcore.TokenCredential } // NewAzureEventHubs returns a new Azure Event hubs instance. @@ -100,23 +92,12 @@ func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { aeh.logger.Info("connecting to Azure Event Hub using Azure AD; the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored") - /*if aeh.metadata.EnableEntityManagement { - if err := aeh.validateEnitityManagementMetadata(); err != nil { - return err - } - - // Create hubManager for eventHub management with AAD. - if managerCreateErr := aeh.createHubManager(); managerCreateErr != nil { - return managerCreateErr - } - - // Get Azure Management plane settings for creating consumer groups using event hubs management client. - settings, err := azauth.NewEnvironmentSettings("azure", metadata.Properties) + if aeh.metadata.EnableEntityManagement { + err = aeh.initEntityManagement(metadata.Properties) if err != nil { - return err + return fmt.Errorf("failed to initialize entity manager: %w", err) } - aeh.managementSettings = settings - }*/ + } } return nil @@ -338,7 +319,10 @@ func (aeh *AzureEventHubs) getProducerClientForTopic(ctx context.Context, topic // Create a new entity if needed if aeh.metadata.EnableEntityManagement { - // TODO: Create a new entity + err = aeh.ensureEventHubEntity(ctx, topic) + if err != nil { + return nil, fmt.Errorf("failed to create Event Hub entity %s: %w", topic, err) + } } clientOpts := &azeventhubs.ProducerClientOptions{ @@ -379,7 +363,22 @@ func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic strin // Create a new entity if needed if aeh.metadata.EnableEntityManagement { - // TODO: Create a new entity + // First ensure that the Event Hub entity exists + // We need to acquire a lock on producers, as creating a producer can perform the same operations + aeh.producersLock.Lock() + err = aeh.ensureEventHubEntity(ctx, topic) + aeh.producersLock.Unlock() + if err != nil { + return nil, fmt.Errorf("failed to create Event Hub entity %s: %w", topic, err) + } + + // Abuse on the lock on checkpoints which are used by all tasks creating processors + aeh.checkpointStoreLock.Lock() + err = aeh.ensureSubscription(ctx, topic) + aeh.checkpointStoreLock.Unlock() + if err != nil { + return nil, fmt.Errorf("failed to create Event Hub subscription to entity %s: %w", topic, err) + } } // Create a consumer client diff --git a/pubsub/azure/eventhubs/eventhubs.go.old b/pubsub/azure/eventhubs/eventhubs.go.old index c7e67d4c0..0c50b0758 100644 --- a/pubsub/azure/eventhubs/eventhubs.go.old +++ b/pubsub/azure/eventhubs/eventhubs.go.old @@ -203,7 +203,6 @@ func validateAndGetHubName(connectionString string) (string, error) { func (aeh *AzureEventHubs) ensureEventHub(ctx context.Context, hubName string) error { if aeh.hubManager == nil { - aeh.logger.Errorf("hubManager client not initialized properly") return fmt.Errorf("hubManager client not initialized properly") } entity, err := aeh.getHubEntity(ctx, hubName) diff --git a/pubsub/azure/eventhubs/eventhubs_test.go b/pubsub/azure/eventhubs/eventhubs_test.go index ceb61cc59..2f300d3e3 100644 --- a/pubsub/azure/eventhubs/eventhubs_test.go +++ b/pubsub/azure/eventhubs/eventhubs_test.go @@ -14,7 +14,6 @@ limitations under the License. package eventhubs import ( - "fmt" "testing" "github.com/stretchr/testify/assert" @@ -32,9 +31,9 @@ func TestParseEventHubsMetadata(t *testing.T) { props := map[string]string{"connectionString": "fake"} metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} - m, err := parseEventHubsMetadata(metadata) + m, err := parseEventHubsMetadata(metadata, testLogger) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "fake", m.ConnectionString) }) @@ -42,240 +41,33 @@ func TestParseEventHubsMetadata(t *testing.T) { props := map[string]string{"eventHubNamespace": "fake"} metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} - m, err := parseEventHubsMetadata(metadata) + m, err := parseEventHubsMetadata(metadata, testLogger) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "fake", m.EventHubNamespace) }) t.Run("test both connectionString and eventHubNamespace given", func(t *testing.T) { - props := map[string]string{"connectionString": "fake", "eventHubNamespace": "fake"} + props := map[string]string{ + "connectionString": "fake", + "eventHubNamespace": "fake", + } metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} - _, err := parseEventHubsMetadata(metadata) + _, err := parseEventHubsMetadata(metadata, testLogger) - assert.Error(t, err) - assert.Equal(t, bothConnectionStringNamespaceErrorMsg, err.Error()) + require.Error(t, err) + assert.ErrorContains(t, err, "only one of connectionString or eventHubNamespace should be passed") }) t.Run("test missing metadata", func(t *testing.T) { props := map[string]string{} metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} - _, err := parseEventHubsMetadata(metadata) + _, err := parseEventHubsMetadata(metadata, testLogger) - assert.Error(t, err) - assert.Equal(t, missingConnectionStringNamespaceErrorMsg, err.Error()) - }) -} - -func TestValidateSubscriptionAttributes(t *testing.T) { - t.Run("test valid configuration", func(t *testing.T) { - props := map[string]string{"connectionString": "fake", "consumerID": "fake", "storageAccountName": "account", "storageAccountKey": "key", "storageContainerName": "container"} - - metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} - m, err := parseEventHubsMetadata(metadata) - - assert.NoError(t, err) - aeh := &AzureEventHubs{logger: testLogger, metadata: m} - assert.Equal(t, m.ConnectionString, "fake") - assert.Equal(t, m.StorageAccountName, "account") - assert.Equal(t, m.StorageAccountKey, "key") - assert.Equal(t, m.StorageContainerName, "container") - assert.Equal(t, m.ConsumerGroup, "fake") - - err = aeh.validateSubscriptionAttributes() - - assert.NoError(t, err) - }) - - type invalidConfigTestCase struct { - name string - config map[string]string - errMsg string - } - invalidConfigTestCases := []invalidConfigTestCase{ - { - "missing consumerID", - map[string]string{"connectionString": "fake", "storageAccountName": "account", "storageAccountKey": "key", "storageContainerName": "container"}, - missingConsumerIDErrorMsg, - }, - { - "missing storageAccountName", - map[string]string{"consumerID": "fake", "connectionString": "fake", "storageAccountKey": "key", "storageContainerName": "container"}, - missingStorageAccountNameErrorMsg, - }, - { - "missing storageAccountKey", - map[string]string{"consumerID": "fake", "connectionString": "fake", "storageAccountName": "name", "storageContainerName": "container"}, - missingStorageAccountKeyErrorMsg, - }, - { - "missing storageContainerName", - map[string]string{"consumerID": "fake", "connectionString": "fake", "storageAccountName": "name", "storageAccountKey": "key"}, - missingStorageContainerNameErrorMsg, - }, - } - - for _, c := range invalidConfigTestCases { - t.Run(c.name, func(t *testing.T) { - metadata := pubsub.Metadata{Base: metadata.Base{Properties: c.config}} - m, err := parseEventHubsMetadata(metadata) - aeh := &AzureEventHubs{logger: testLogger, metadata: m} - require.NoError(t, err) - err = aeh.validateSubscriptionAttributes() - assert.Error(t, err) - assert.Equal(t, c.errMsg, err.Error()) - }) - } -} - -func TestValidateEnitityManagementMetadata(t *testing.T) { - t.Run("test valid configuration", func(t *testing.T) { - props := map[string]string{"eventHubNamespace": "fake", "messageRetentionInDays": "2", "partitionCount": "3", "resourceGroupName": "rg", "subscriptionID": "id"} - - metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} - m, err := parseEventHubsMetadata(metadata) - - require.NoError(t, err) - aeh := &AzureEventHubs{logger: testLogger, metadata: m} - assert.Equal(t, "fake", m.EventHubNamespace) - assert.Equal(t, int32(2), m.MessageRetentionInDays) - assert.Equal(t, int32(3), m.PartitionCount) - - err = aeh.validateEnitityManagementMetadata() - assert.NoError(t, err) - assert.Equal(t, int32(2), m.MessageRetentionInDays) - assert.Equal(t, int32(3), m.PartitionCount) - assert.Equal(t, "rg", m.ResourceGroupName) - assert.Equal(t, "id", m.SubscriptionID) - }) - - t.Run("test valid configuration", func(t *testing.T) { - props := map[string]string{"eventHubNamespace": "fake", "resourceGroupName": "rg", "subscriptionID": "id"} - - metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} - m, err := parseEventHubsMetadata(metadata) - - require.NoError(t, err) - aeh := &AzureEventHubs{logger: testLogger, metadata: m} - assert.Equal(t, "fake", m.EventHubNamespace) - assert.Equal(t, int32(0), m.MessageRetentionInDays) - assert.Equal(t, int32(0), m.PartitionCount) - - err = aeh.validateEnitityManagementMetadata() - assert.NoError(t, err) - assert.Equal(t, int32(1), m.MessageRetentionInDays) - assert.Equal(t, int32(1), m.PartitionCount) - assert.Equal(t, "rg", m.ResourceGroupName) - assert.Equal(t, "id", m.SubscriptionID) - }) - - type invalidConfigTestCase struct { - name string - config map[string]string - messageRetentionInDays int32 - partitionCount int32 - errMsg string - } - invalidConfigTestCases := []invalidConfigTestCase{ - { - "negative message rentention days", - map[string]string{"eventHubNamespace": "fake", "messageRetentionInDays": "-2", "resourceGroupName": "rg", "subscriptionID": "id"}, - defaultMessageRetentionInDays, - defaultPartitionCount, - "", - }, - { - "more than max message rentention days", - map[string]string{"eventHubNamespace": "fake", "messageRetentionInDays": "91", "resourceGroupName": "rg", "subscriptionID": "id"}, - defaultMessageRetentionInDays, - defaultPartitionCount, - "", - }, - { - "negative partition count", - map[string]string{"eventHubNamespace": "fake", "partitionCount": "-2", "resourceGroupName": "rg", "subscriptionID": "id"}, - defaultMessageRetentionInDays, - defaultPartitionCount, - "", - }, - { - "more than max partition count", - map[string]string{"eventHubNamespace": "fake", "partitionCount": "1030", "resourceGroupName": "rg", "subscriptionID": "id"}, - defaultMessageRetentionInDays, - defaultPartitionCount, - "", - }, - { - "missingResourceGroupName", - map[string]string{"eventHubNamespace": "fake", "subscriptionID": "id"}, - defaultMessageRetentionInDays, - defaultPartitionCount, - missingResourceGroupNameMsg, - }, - { - "missingSubscriptionID", - map[string]string{"eventHubNamespace": "fake", "resourceGroupName": "id"}, - defaultMessageRetentionInDays, - defaultPartitionCount, - missingSubscriptionIDMsg, - }, - } - - for _, c := range invalidConfigTestCases { - t.Run(c.name, func(t *testing.T) { - metadata := pubsub.Metadata{Base: metadata.Base{Properties: c.config}} - m, err := parseEventHubsMetadata(metadata) - aeh := &AzureEventHubs{logger: testLogger, metadata: m} - require.NoError(t, err) - err = aeh.validateEnitityManagementMetadata() - if c.errMsg != "" { - assert.Error(t, err) - assert.Equal(t, c.errMsg, err.Error()) - } else { - assert.NoError(t, err) - } - assert.Equal(t, c.messageRetentionInDays, aeh.metadata.MessageRetentionInDays) - assert.Equal(t, c.partitionCount, aeh.metadata.PartitionCount) - }) - } -} - -func TestGetStoragePrefixString(t *testing.T) { - props := map[string]string{"connectionString": "fake", "consumerID": "test"} - - metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} - m, err := parseEventHubsMetadata(metadata) - - require.NoError(t, err) - - aeh := &AzureEventHubs{logger: testLogger, metadata: m} - - actual := aeh.getStoragePrefixString("topic") - - assert.Equal(t, "dapr-topic-test-", actual) -} - -func TestValidateAndGetHubName(t *testing.T) { - t.Run("valid connectionString with hub name", func(t *testing.T) { - connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key;EntityPath=testHub" - h, err := validateAndGetHubName(connectionString) - assert.NoError(t, err) - assert.Equal(t, "testHub", h) - }) - - t.Run("valid connectionString without hub name", func(t *testing.T) { - connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key" - h, err := validateAndGetHubName(connectionString) - assert.NoError(t, err) - assert.Empty(t, h) - }) - - t.Run("invalid connectionString ", func(t *testing.T) { - connectionString := "Endpoint=sb://fake.servicebus.windows.net/;ShareKeyName=fakeKey;SharedAccessKey=key" - _, err := validateAndGetHubName(connectionString) - assert.Error(t, err) + require.Error(t, err) + assert.ErrorContains(t, err, "one of connectionString or eventHubNamespace is required") }) } @@ -284,41 +76,64 @@ func TestConstructConnectionStringFromTopic(t *testing.T) { connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key" topic := "testHub" - aeh := &AzureEventHubs{logger: testLogger, metadata: &azureEventHubsMetadata{ConnectionString: connectionString}} + metadata := pubsub.Metadata{Base: metadata.Base{ + Properties: map[string]string{ + "connectionString": connectionString, + }, + }} + aeh := &AzureEventHubs{logger: testLogger} + err := aeh.Init(metadata) + require.NoError(t, err) c, err := aeh.constructConnectionStringFromTopic(topic) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, connectionString+";EntityPath=testHub", c) }) t.Run("valid connectionString with hub name", func(t *testing.T) { connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key;EntityPath=testHub" topic := "testHub" - aeh := &AzureEventHubs{logger: testLogger, metadata: &azureEventHubsMetadata{ConnectionString: connectionString}} + metadata := pubsub.Metadata{Base: metadata.Base{ + Properties: map[string]string{ + "connectionString": connectionString, + }, + }} + aeh := &AzureEventHubs{logger: testLogger} + err := aeh.Init(metadata) + require.NoError(t, err) c, err := aeh.constructConnectionStringFromTopic(topic) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, connectionString, c) }) t.Run("invalid connectionString with hub name", func(t *testing.T) { - connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;ShareKey=key;EntityPath=testHub" - topic := "testHub" + connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;NoKey=key;EntityPath=testHub" - aeh := &AzureEventHubs{logger: testLogger, metadata: &azureEventHubsMetadata{ConnectionString: connectionString}} - - c, err := aeh.constructConnectionStringFromTopic(topic) - assert.Error(t, err) - assert.Equal(t, "", c) + metadata := pubsub.Metadata{Base: metadata.Base{ + Properties: map[string]string{ + "connectionString": connectionString, + }, + }} + aeh := &AzureEventHubs{logger: testLogger} + err := aeh.Init(metadata) + require.Error(t, err) }) t.Run("valid connectionString with different hub name", func(t *testing.T) { connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key;EntityPath=testHub" topic := "differentHub" - aeh := &AzureEventHubs{logger: testLogger, metadata: &azureEventHubsMetadata{ConnectionString: connectionString}} + metadata := pubsub.Metadata{Base: metadata.Base{ + Properties: map[string]string{ + "connectionString": connectionString, + }, + }} + aeh := &AzureEventHubs{logger: testLogger} + err := aeh.Init(metadata) + require.NoError(t, err) c, err := aeh.constructConnectionStringFromTopic(topic) - assert.Error(t, err) - assert.Equal(t, (fmt.Sprintf(differentTopicConnectionStringErrorTmpl, topic)), err.Error()) + require.Error(t, err) + assert.ErrorContains(t, err, "does not match the Event Hub name in the connection string") assert.Equal(t, "", c) }) } From 9729c155c19835bf9eb30b540a7ee6ea9507f653 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 00:47:24 +0000 Subject: [PATCH 09/42] Retry failed messages + other fixes Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- go.mod | 2 +- pubsub/azure/eventhubs/entity_management.go | 6 +-- pubsub/azure/eventhubs/eventhubs.go | 48 +++++++++++++++++-- .../pubsub/azure/eventhubs/go.mod | 1 + .../pubsub/azure/eventhubs/go.sum | 2 +- 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 962dd3044..e2db7c12a 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd @@ -144,7 +145,6 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect diff --git a/pubsub/azure/eventhubs/entity_management.go b/pubsub/azure/eventhubs/entity_management.go index 85b8f99cc..5cd226f44 100644 --- a/pubsub/azure/eventhubs/entity_management.go +++ b/pubsub/azure/eventhubs/entity_management.go @@ -160,9 +160,9 @@ func (aeh *AzureEventHubs) ensureSubscription(parentCtx context.Context, hubName backOffConfig.MaxRetries = resourceCheckMaxRetry b := backOffConfig.NewBackOffWithContext(parentCtx) create, err := retry.NotifyRecoverWithData(func() (bool, error) { - c, err := aeh.shouldCreateConsumerGroup(parentCtx, client, hubName) - if err != nil { - return false, err + c, cErr := aeh.shouldCreateConsumerGroup(parentCtx, client, hubName) + if cErr != nil { + return false, cErr } return c, nil }, b, func(_ error, _ time.Duration) { diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index edf4f6dde..201fcddd0 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -17,7 +17,9 @@ import ( "context" "errors" "fmt" + "strconv" "sync" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" @@ -30,6 +32,7 @@ import ( "github.com/dapr/components-contrib/pubsub" "github.com/dapr/components-contrib/pubsub/azure/eventhubs/conn" "github.com/dapr/kit/logger" + "github.com/dapr/kit/retry" ) // AzureEventHubs allows sending/receiving Azure Event Hubs events. @@ -37,6 +40,7 @@ type AzureEventHubs struct { metadata *azureEventHubsMetadata logger logger.Logger + backOffConfig retry.Config producersLock *sync.RWMutex producers map[string]*azeventhubs.ProducerClient checkpointStoreCache azeventhubs.CheckpointStore @@ -100,6 +104,12 @@ func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { } } + // Default retry configuration is used if no backOff properties are set. + err = retry.DecodeConfigWithPrefix(&aeh.backOffConfig, metadata.Properties, "backOff") + if err != nil { + return fmt.Errorf("failed to decode backoff configuration") + } + return nil } @@ -168,8 +178,38 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su return fmt.Errorf("error trying to establish a connection: %w", err) } + // This component has built-in retries because Event Hubs doesn't support N/ACK for messages + retryHandler := func(ctx context.Context, msg *pubsub.NewMessage) error { + b := aeh.backOffConfig.NewBackOffWithContext(subscribeCtx) + + mID := msg.Metadata[sysPropMessageID] + if mID == "" { + mID = "(nil)" + } + // This method is synchronous so no risk of race conditions if using side effects + var attempts int + retryerr := retry.NotifyRecover(func() error { + attempts++ + aeh.logger.Debugf("Processing EventHubs event %s/%s (attempt: %d)", msg.Topic, mID, attempts) + + if attempts > 1 { + msg.Metadata["dapr-attempt"] = strconv.Itoa(attempts) + } + + return handler(ctx, msg) + }, b, func(_ error, _ time.Duration) { + aeh.logger.Warnf("Error processing EventHubs event: %s/%s. Retrying...", msg.Topic, mID) + }, func() { + aeh.logger.Warnf("Successfully processed EventHubs event after it previously failed: %s/%s", msg.Topic, mID) + }) + if retryerr != nil { + aeh.logger.Errorf("Too many failed attempts at processing Eventhubs event: %s/%s. Error: %v", msg.Topic, mID, err) + } + return retryerr + } + // Get the subscribe handler - eventHandler := subscribeHandler(subscribeCtx, req.Topic, getAllProperties, handler) + eventHandler := subscribeHandler(subscribeCtx, req.Topic, getAllProperties, retryHandler) // Process all partition clients as they come in go func() { @@ -248,10 +288,8 @@ func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, topic str if len(events) != 0 { for _, event := range events { - // TODO: Handle errors - // Maybe consider exiting this method (without updating the checkpoint) so another app will re-try it? - err := eventHandler(event) - fmt.Printf("EVENT PROCESSED - err: %v\n", err) + // Process the event in its own goroutine + go eventHandler(event) } // Update the checkpoint with the last event received. If we lose ownership of this partition or have to restart the next owner will start from this point. diff --git a/tests/certification/pubsub/azure/eventhubs/go.mod b/tests/certification/pubsub/azure/eventhubs/go.mod index f513ce7fa..b43089b17 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.mod +++ b/tests/certification/pubsub/azure/eventhubs/go.mod @@ -22,6 +22,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/pubsub/azure/eventhubs/go.sum b/tests/certification/pubsub/azure/eventhubs/go.sum index 2e8cf1d14..6a957384b 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.sum +++ b/tests/certification/pubsub/azure/eventhubs/go.sum @@ -40,7 +40,6 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= @@ -50,6 +49,7 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLC github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 h1:X/ePaAG8guM7j5WORT5eEIw7cGUxe9Ah1jEQJKLmmSo= github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0/go.mod h1:5dog28UP3dd1BnCPFYvyHfsmA+Phmoezt+KWT5cZnyc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 h1:BWeAAEzkCnL0ABVJqs+4mYudNch7oFGPtTlSmIWL8ms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0/go.mod h1:Y3gnVwfaz8h6L1YHar+NfWORtBoVUSB5h4GlGkdeF7Q= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= From bf1447fc100ec24d594d674c45ee43ceedf9c3c9 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 00:58:39 +0000 Subject: [PATCH 10/42] Fixed setup-azure-conf-test due to changes in Azure CLI Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../conformance/azure/setup-azure-conf-test.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh index feca0a7d3..498f3afe0 100755 --- a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh +++ b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh @@ -228,7 +228,7 @@ STORAGE_CONTAINER_VAR_NAME="AzureBlobStorageContainer" STORAGE_QUEUE_VAR_NAME="AzureBlobStorageQueue" # Derived variables -ADMIN_ID="$(az ad user list --filter "userPrincipalName eq '${ADMIN_UPN}'" --query "[].objectId" --output tsv)" +ADMIN_ID="$(az ad user list --filter "userPrincipalName eq '${ADMIN_UPN}'" --query "[].id" --output tsv)" if [[ -z "${ADMIN_ID}" ]]; then echo "Could not find user with upn ${ADMIN_UPN}" exit 1 @@ -245,8 +245,8 @@ az config set extension.use_dynamic_install=yes_without_prompt # Create Service Principals for use with the conformance tests CERT_AUTH_SP_NAME="${PREFIX}-akv-conf-test-sp" -az ad sp create-for-rbac --name "${CERT_AUTH_SP_NAME}" --skip-assignment --years 1 -CERT_AUTH_SP_ID="$(az ad sp list --display-name "${CERT_AUTH_SP_NAME}" --query "[].objectId" --output tsv)" +az ad sp create-for-rbac --name "${CERT_AUTH_SP_NAME}" --years 1 +CERT_AUTH_SP_ID="$(az ad sp list --display-name "${CERT_AUTH_SP_NAME}" --query "[].id" --output tsv)" echo "Created Service Principal for cert auth: ${CERT_AUTH_SP_NAME}" if [[ -n ${CREDENTIALS_PATH} ]]; then @@ -258,13 +258,13 @@ if [[ -n ${CREDENTIALS_PATH} ]]; then exit 1 fi SDK_AUTH_SP_NAME="$(az ad sp show --id "${SDK_AUTH_SP_APPID}" --query "appDisplayName" --output tsv)" - SDK_AUTH_SP_ID="$(az ad sp show --id "${SDK_AUTH_SP_APPID}" --query "objectId" --output tsv)" + SDK_AUTH_SP_ID="$(az ad sp show --id "${SDK_AUTH_SP_APPID}" --query "id" --output tsv)" echo "Using Service Principal from ${CREDENTIALS_PATH} for SDK Auth: ${SDK_AUTH_SP_NAME}" else SDK_AUTH_SP_NAME="${PREFIX}-conf-test-runner-sp" - SDK_AUTH_SP_INFO="$(az ad sp create-for-rbac --name "${SDK_AUTH_SP_NAME}" --sdk-auth --skip-assignment --years 1)" + SDK_AUTH_SP_INFO="$(az ad sp create-for-rbac --name "${SDK_AUTH_SP_NAME}" --sdk-auth --years 1)" SDK_AUTH_SP_CLIENT_SECRET="$(echo "${SDK_AUTH_SP_INFO}" | grep 'clientSecret' | sed -E 's/(.*clientSecret\"\: \")|\".*//g')" - SDK_AUTH_SP_ID="$(az ad sp list --display-name "${SDK_AUTH_SP_NAME}" --query "[].objectId" --output tsv)" + SDK_AUTH_SP_ID="$(az ad sp list --display-name "${SDK_AUTH_SP_NAME}" --query "[].id" --output tsv)" echo "${SDK_AUTH_SP_INFO}" echo "Created Service Principal for SDK Auth: ${SDK_AUTH_SP_NAME}" AZURE_CREDENTIALS_FILENAME="${OUTPUT_PATH}/AZURE_CREDENTIALS" @@ -378,7 +378,7 @@ az keyvault set-policy --name "${KEYVAULT_NAME}" -g "${RESOURCE_GROUP_NAME}" --s # Creating service principal for service principal authentication with KeyVault AKV_SPAUTH_SP_NAME="${PREFIX}-akv-spauth-conf-test-sp" echo "Creating service principal ${AKV_SPAUTH_SP_NAME} for use with KeyVault ${KEYVAULT_NAME}" -{ read AKV_SPAUTH_SP_CLIENT_ID ; read AKV_SPAUTH_SP_CLIENT_SECRET ; } < <(az ad sp create-for-rbac --name ${AKV_SPAUTH_SP_NAME} --skip-assignment --years 1 --query "[appId,password]" -otsv) +{ read AKV_SPAUTH_SP_CLIENT_ID ; read AKV_SPAUTH_SP_CLIENT_SECRET ; } < <(az ad sp create-for-rbac --name ${AKV_SPAUTH_SP_NAME} --years 1 --query "[appId,password]" -otsv) # Give the service principal read access to the KeyVault Secrets AKV_SPAUTH_SP_OBJECTID="$(az ad sp show --id ${AKV_SPAUTH_SP_CLIENT_ID} --query objectId -otsv)" @@ -716,7 +716,7 @@ az keyvault secret set --name "${IOT_HUB_PUBSUB_CONSUMER_GROUP_VAR_NAME}" --vaul # CERTIFICATION TESTS: Create service principal and grant resource access # ------------------------------------------------------------------------ CERTIFICATION_SPAUTH_SP_NAME="${PREFIX}-certification-spauth-conf-test-sp" -{ read CERTIFICATION_SPAUTH_SP_CLIENT_ID ; read CERTIFICATION_SPAUTH_SP_CLIENT_SECRET ; } < <(az ad sp create-for-rbac --name ${CERTIFICATION_SPAUTH_SP_NAME} --skip-assignment --years 1 --query "[appId,password]" -otsv) +{ read CERTIFICATION_SPAUTH_SP_CLIENT_ID ; read CERTIFICATION_SPAUTH_SP_CLIENT_SECRET ; } < <(az ad sp create-for-rbac --name ${CERTIFICATION_SPAUTH_SP_NAME} --years 1 --query "[appId,password]" -otsv) CERTIFICATION_SPAUTH_SP_PRINCIPAL_ID="$(az ad sp list --display-name "${CERTIFICATION_SPAUTH_SP_NAME}" --query "[].objectId" --output tsv)" # Give the service principal used for certification test access to the relevant data plane resources From 8c63bd84cb0b2c6dbc733c41056a1d96f2d9e807 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 01:05:56 +0000 Subject: [PATCH 11/42] More fixes to script Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../infrastructure/conformance/azure/setup-azure-conf-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh index 498f3afe0..c64477a8f 100755 --- a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh +++ b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh @@ -381,7 +381,7 @@ echo "Creating service principal ${AKV_SPAUTH_SP_NAME} for use with KeyVault ${K { read AKV_SPAUTH_SP_CLIENT_ID ; read AKV_SPAUTH_SP_CLIENT_SECRET ; } < <(az ad sp create-for-rbac --name ${AKV_SPAUTH_SP_NAME} --years 1 --query "[appId,password]" -otsv) # Give the service principal read access to the KeyVault Secrets -AKV_SPAUTH_SP_OBJECTID="$(az ad sp show --id ${AKV_SPAUTH_SP_CLIENT_ID} --query objectId -otsv)" +AKV_SPAUTH_SP_OBJECTID="$(az ad sp show --id ${AKV_SPAUTH_SP_CLIENT_ID} --query id -otsv)" az keyvault set-policy --name "${KEYVAULT_NAME}" -g "${RESOURCE_GROUP_NAME}" --secret-permissions get list --object-id "${AKV_SPAUTH_SP_OBJECTID}" # Update service principal credentials and roles for created resources @@ -717,7 +717,7 @@ az keyvault secret set --name "${IOT_HUB_PUBSUB_CONSUMER_GROUP_VAR_NAME}" --vaul # ------------------------------------------------------------------------ CERTIFICATION_SPAUTH_SP_NAME="${PREFIX}-certification-spauth-conf-test-sp" { read CERTIFICATION_SPAUTH_SP_CLIENT_ID ; read CERTIFICATION_SPAUTH_SP_CLIENT_SECRET ; } < <(az ad sp create-for-rbac --name ${CERTIFICATION_SPAUTH_SP_NAME} --years 1 --query "[appId,password]" -otsv) -CERTIFICATION_SPAUTH_SP_PRINCIPAL_ID="$(az ad sp list --display-name "${CERTIFICATION_SPAUTH_SP_NAME}" --query "[].objectId" --output tsv)" +CERTIFICATION_SPAUTH_SP_PRINCIPAL_ID="$(az ad sp list --display-name "${CERTIFICATION_SPAUTH_SP_NAME}" --query "[].id" --output tsv)" # Give the service principal used for certification test access to the relevant data plane resources # Cosmos DB From 63621baaef663cdfb710fd97c04f773e935bec33 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 03:46:25 +0000 Subject: [PATCH 12/42] And yet some more Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../conformance/azure/setup-azure-conf-test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh index c64477a8f..6d6538beb 100755 --- a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh +++ b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh @@ -251,8 +251,8 @@ echo "Created Service Principal for cert auth: ${CERT_AUTH_SP_NAME}" if [[ -n ${CREDENTIALS_PATH} ]]; then SDK_AUTH_SP_INFO="$(cat ${CREDENTIALS_PATH})" - SDK_AUTH_SP_APPID="$(echo "${SDK_AUTH_SP_INFO}" | grep 'clientId' | sed -E 's/(.*clientId\"\: \")|\",//g')" - SDK_AUTH_SP_CLIENT_SECRET="$(echo "${SDK_AUTH_SP_INFO}" | grep 'clientSecret' | sed -E 's/(.*clientSecret\"\: \")|\",//g')" + SDK_AUTH_SP_APPID="$(echo "${SDK_AUTH_SP_INFO}" | jq -r '.clientId')" + SDK_AUTH_SP_CLIENT_SECRET="$(echo "${SDK_AUTH_SP_INFO}" | jq -r '.clientSecret')" if [[ -z ${SDK_AUTH_SP_APPID} || -z ${SDK_AUTH_SP_CLIENT_SECRET} ]]; then echo "Invalid credentials JSON file. Contents should match output of 'az ad sp create-for-rbac' command." exit 1 @@ -263,7 +263,7 @@ if [[ -n ${CREDENTIALS_PATH} ]]; then else SDK_AUTH_SP_NAME="${PREFIX}-conf-test-runner-sp" SDK_AUTH_SP_INFO="$(az ad sp create-for-rbac --name "${SDK_AUTH_SP_NAME}" --sdk-auth --years 1)" - SDK_AUTH_SP_CLIENT_SECRET="$(echo "${SDK_AUTH_SP_INFO}" | grep 'clientSecret' | sed -E 's/(.*clientSecret\"\: \")|\".*//g')" + SDK_AUTH_SP_CLIENT_SECRET="$(echo "${SDK_AUTH_SP_INFO}" | jq -r '.clientSecret')" SDK_AUTH_SP_ID="$(az ad sp list --display-name "${SDK_AUTH_SP_NAME}" --query "[].id" --output tsv)" echo "${SDK_AUTH_SP_INFO}" echo "Created Service Principal for SDK Auth: ${SDK_AUTH_SP_NAME}" @@ -386,7 +386,7 @@ az keyvault set-policy --name "${KEYVAULT_NAME}" -g "${RESOURCE_GROUP_NAME}" --s # Update service principal credentials and roles for created resources echo "Creating ${CERT_AUTH_SP_NAME} certificate ..." -az ad sp credential reset --name "${CERT_AUTH_SP_NAME}" --create-cert --cert "${KEYVAULT_CERT_NAME}" --keyvault "${KEYVAULT_NAME}" +az ad sp credential reset --id "${CERT_AUTH_SP_ID}" --create-cert --cert "${KEYVAULT_CERT_NAME}" --keyvault "${KEYVAULT_NAME}" # Add an EventGrid role to the SDK auth Service Principal so that it can be reused for the EventGrid binding conformance tests. EVENT_GRID_SCOPE="/subscriptions/${SUB_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.EventGrid/topics/${EVENT_GRID_TOPIC_NAME}" @@ -499,7 +499,7 @@ echo export ${KEYVAULT_CERT_NAME}=\"${KEYVAULT_CERT_FILE}\" >> "${ENV_CONFIG_FIL echo export ${KEYVAULT_NAME_VAR_NAME}=\"${KEYVAULT_NAME}\" >> "${ENV_CONFIG_FILENAME}" az keyvault secret set --name "${KEYVAULT_NAME_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${KEYVAULT_NAME}" -KEYVAULT_TENANT_ID="$(az ad sp list --display-name "${CERT_AUTH_SP_NAME}" --query "[].appOwnerTenantId" --output tsv)" +KEYVAULT_TENANT_ID="$(az ad sp list --display-name "${CERT_AUTH_SP_NAME}" --query "[].appOwnerOrganizationId" --output tsv)" echo export ${KEYVAULT_TENANT_ID_VAR_NAME}=\"${KEYVAULT_TENANT_ID}\" >> "${ENV_CONFIG_FILENAME}" az keyvault secret set --name "${KEYVAULT_TENANT_ID_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${KEYVAULT_TENANT_ID}" @@ -735,7 +735,7 @@ ASB_ID=$(az servicebus namespace show --resource-group "${RESOURCE_GROUP_NAME}" az role assignment create --assignee "${CERTIFICATION_SPAUTH_SP_PRINCIPAL_ID}" --role "Azure Service Bus Data Owner" --scope "${ASB_ID}" # Now export the service principal information -CERTIFICATION_TENANT_ID="$(az ad sp list --display-name "${CERTIFICATION_SPAUTH_SP_NAME}" --query "[].appOwnerTenantId" --output tsv)" +CERTIFICATION_TENANT_ID="$(az ad sp list --display-name "${CERTIFICATION_SPAUTH_SP_NAME}" --query "[].appOwnerOrganizationId" --output tsv)" echo export ${CERTIFICATION_SERVICE_PRINCIPAL_CLIENT_ID_VAR_NAME}=\"${CERTIFICATION_SPAUTH_SP_CLIENT_ID}\" >> "${ENV_CONFIG_FILENAME}" az keyvault secret set --name "${CERTIFICATION_SERVICE_PRINCIPAL_CLIENT_ID_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${CERTIFICATION_SPAUTH_SP_CLIENT_ID}" echo export ${CERTIFICATION_SERVICE_PRINCIPAL_CLIENT_SECRET_VAR_NAME}=\"${CERTIFICATION_SPAUTH_SP_CLIENT_SECRET}\" >> "${ENV_CONFIG_FILENAME}" From 885fb5a2d48582573137f3cde321f09e1352e407 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 04:15:18 +0000 Subject: [PATCH 13/42] Re-implemented BulkPublish Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 90 +++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 17 deletions(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 201fcddd0..77272c0d8 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -29,6 +29,7 @@ import ( azauth "github.com/dapr/components-contrib/internal/authentication/azure" "github.com/dapr/components-contrib/internal/utils" + contribMetadata "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/components-contrib/pubsub/azure/eventhubs/conn" "github.com/dapr/kit/logger" @@ -123,31 +124,91 @@ func (aeh *AzureEventHubs) Publish(ctx context.Context, req *pubsub.PublishReque return errors.New("parameter 'topic' is required") } + // Get the partition key and create the batch of messages + batchOpts := &azeventhubs.EventDataBatchOptions{} + if pk := req.Metadata["partitionKey"]; pk != "" { + batchOpts.PartitionKey = &pk + } + messages := []*azeventhubs.EventData{ + { + Body: req.Data, + ContentType: req.ContentType, + }, + } + + // Publish the message + return aeh.doPublish(ctx, req.Topic, messages, batchOpts) +} + +// BulkPublish sends data to Azure Event Hubs in bulk. +func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPublishRequest) (pubsub.BulkPublishResponse, error) { + res := pubsub.BulkPublishResponse{} + + if req.Topic == "" { + return res, errors.New("parameter 'topic' is required") + } + + // Batch options + batchOpts := &azeventhubs.EventDataBatchOptions{} + if val := req.Metadata[contribMetadata.MaxBulkPubBytesKey]; val != "" { + maxBytes, err := strconv.ParseUint(val, 10, 63) + if err == nil && maxBytes > 0 { + batchOpts.MaxBytes = maxBytes + } + } + + // Build the batch of messages + messages := make([]*azeventhubs.EventData, len(req.Entries)) + for i, entry := range req.Entries { + messages[i] = &azeventhubs.EventData{ + Body: entry.Event, + } + if entry.ContentType != "" { + messages[i].ContentType = &entry.ContentType + } + if val := entry.Metadata["partitionKey"]; val != "" { + if batchOpts.PartitionKey != nil && *batchOpts.PartitionKey != val { + return res, errors.New("cannot send messages to different partitions") + } + batchOpts.PartitionKey = &val + } + } + + // Publish the message + err := aeh.doPublish(ctx, req.Topic, messages, batchOpts) + if err != nil { + // Partial success is not supported by Azure Event Hubs. + // If an error occurs, all events are considered failed. + return pubsub.NewBulkPublishResponse(req.Entries, err), err + } + + return res, nil +} + +// Internal method used by Publish and BulkPublish to send messages +func (aeh *AzureEventHubs) doPublish(ctx context.Context, topic string, messages []*azeventhubs.EventData, batchOpts *azeventhubs.EventDataBatchOptions) error { // Get the producer client - client, err := aeh.getProducerClientForTopic(ctx, req.Topic) + client, err := aeh.getProducerClientForTopic(ctx, topic) if err != nil { return fmt.Errorf("error trying to establish a connection: %w", err) } // Build the batch of messages - batchOpts := &azeventhubs.EventDataBatchOptions{} - if pk := req.Metadata["partitionKey"]; pk != "" { - batchOpts.PartitionKey = &pk - } batch, err := client.NewEventDataBatch(ctx, batchOpts) if err != nil { return fmt.Errorf("error creating batch: %w", err) } - err = batch.AddEventData(&azeventhubs.EventData{ - Body: req.Data, - ContentType: req.ContentType, - }, nil) - if err != nil { - return fmt.Errorf("error adding message to batch: %w", err) + + // Add all messages + for _, msg := range messages { + err = batch.AddEventData(msg, nil) + if err != nil { + return fmt.Errorf("error adding messages to batch: %w", err) + } } // Send the message - client.SendEventDataBatch(ctx, batch, nil) + err = client.SendEventDataBatch(ctx, batch, nil) if err != nil { return fmt.Errorf("error publishing batch: %w", err) } @@ -155,11 +216,6 @@ func (aeh *AzureEventHubs) Publish(ctx context.Context, req *pubsub.PublishReque return nil } -// BulkPublish sends data to Azure Event Hubs in bulk. -func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPublishRequest) (pubsub.BulkPublishResponse, error) { - return pubsub.BulkPublishResponse{}, nil -} - // Subscribe receives data from Azure Event Hubs. func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) (err error) { if aeh.metadata.ConsumerGroup == "" { From 01e7d4167b4735fa63577b7b1a51d10f2e9f800a Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 04:24:35 +0000 Subject: [PATCH 14/42] Do not retry forever by default Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 77272c0d8..088cc3ab4 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -105,7 +105,10 @@ func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { } } - // Default retry configuration is used if no backOff properties are set. + // Default retry configuration is used if no backOff properties are set + // backOff max retry config is set to 3, which means 3 retries by default + aeh.backOffConfig = retry.DefaultConfig() + aeh.backOffConfig.MaxRetries = 3 err = retry.DecodeConfigWithPrefix(&aeh.backOffConfig, metadata.Properties, "backOff") if err != nil { return fmt.Errorf("failed to decode backoff configuration") From 8d679f06f93e1e8a77519b3a1f5aeec29650b3c7 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 16:59:27 +0000 Subject: [PATCH 15/42] Working on upgrading from track1 SDKs Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../azure/appconfig/appconfig_test.go | 14 +-- go.mod | 6 +- go.sum | 8 +- .../component/azure/blobstorage/client.go | 2 +- pubsub/azure/eventhubs/entity_management.go | 4 +- pubsub/azure/eventhubs/eventhubs.go | 20 +++- pubsub/azure/eventhubs/metadata.go | 4 + pubsub/azure/eventhubs/track1_upgrade.go | 112 ++++++++++++++++++ .../bindings/azure/blobstorage/go.mod | 2 +- .../bindings/azure/blobstorage/go.sum | 4 +- .../bindings/azure/cosmosdb/go.mod | 4 +- .../bindings/azure/cosmosdb/go.sum | 8 +- .../bindings/azure/eventhubs/go.mod | 4 +- .../bindings/azure/eventhubs/go.sum | 8 +- .../bindings/azure/servicebusqueues/go.mod | 2 +- .../bindings/azure/servicebusqueues/go.sum | 4 +- .../bindings/azure/storagequeues/go.mod | 2 +- .../bindings/azure/storagequeues/go.sum | 4 +- .../pubsub/azure/eventhubs/go.mod | 3 +- .../pubsub/azure/eventhubs/go.sum | 6 +- .../pubsub/azure/servicebus/topics/go.mod | 2 +- .../pubsub/azure/servicebus/topics/go.sum | 4 +- .../secretstores/azure/keyvault/go.mod | 2 +- .../secretstores/azure/keyvault/go.sum | 4 +- .../state/azure/blobstorage/go.mod | 2 +- .../state/azure/blobstorage/go.sum | 4 +- .../certification/state/azure/cosmosdb/go.mod | 4 +- .../certification/state/azure/cosmosdb/go.sum | 8 +- .../state/azure/tablestorage/go.mod | 2 +- .../state/azure/tablestorage/go.sum | 4 +- 30 files changed, 197 insertions(+), 60 deletions(-) create mode 100644 pubsub/azure/eventhubs/track1_upgrade.go diff --git a/configuration/azure/appconfig/appconfig_test.go b/configuration/azure/appconfig/appconfig_test.go index 7cff4cad9..933ccea60 100644 --- a/configuration/azure/appconfig/appconfig_test.go +++ b/configuration/azure/appconfig/appconfig_test.go @@ -22,12 +22,12 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" - "github.com/Azure/go-autorest/autorest/to" "github.com/stretchr/testify/assert" "github.com/dapr/components-contrib/configuration" mdata "github.com/dapr/components-contrib/metadata" "github.com/dapr/kit/logger" + "github.com/dapr/kit/ptr" ) type MockConfigurationStore struct { @@ -38,8 +38,8 @@ func (m *MockConfigurationStore) GetSetting(ctx context.Context, key string, opt if key == "testKey" || key == "test_sentinel_key" { settings := azappconfig.Setting{} - settings.Key = to.StringPtr("testKey") - settings.Value = to.StringPtr("testValue") + settings.Key = ptr.Of("testKey") + settings.Value = ptr.Of("testValue") resp := azappconfig.GetSettingResponse{} resp.Setting = settings @@ -53,12 +53,12 @@ func (m *MockConfigurationStore) NewListSettingsPager(selector azappconfig.Setti settings := make([]azappconfig.Setting, 2) setting1 := azappconfig.Setting{} - setting1.Key = to.StringPtr("testKey-1") - setting1.Value = to.StringPtr("testValue-1") + setting1.Key = ptr.Of("testKey-1") + setting1.Value = ptr.Of("testValue-1") setting2 := azappconfig.Setting{} - setting2.Key = to.StringPtr("testKey-2") - setting2.Value = to.StringPtr("testValue-2") + setting2.Key = ptr.Of("testKey-2") + setting2.Value = ptr.Of("testValue-2") settings[0] = setting1 settings[1] = setting2 diff --git a/go.mod b/go.mod index e2db7c12a..3d4c22b85 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( dubbo.apache.org/dubbo-go/v3 v3.0.3-0.20230118042253-4f159a2b38f3 github.com/Azure/azure-amqp-common-go/v4 v4.0.0 github.com/Azure/azure-event-hubs-go/v3 v3.4.0 - github.com/Azure/azure-sdk-for-go v65.0.0+incompatible - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3 @@ -27,7 +27,6 @@ require ( github.com/Azure/go-autorest/autorest v0.11.28 github.com/Azure/go-autorest/autorest/adal v0.9.21 github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 - github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Shopify/sarama v1.38.0 github.com/aerospike/aerospike-client-go v4.5.2+incompatible @@ -148,6 +147,7 @@ require ( github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect diff --git a/go.sum b/go.sum index dd58ee9ab..e69528199 100644 --- a/go.sum +++ b/go.sum @@ -411,11 +411,11 @@ github.com/Azure/azure-event-hubs-go/v3 v3.4.0/go.mod h1:ODgt5C1/c73FToYj+mWojUJ github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= -github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= diff --git a/internal/component/azure/blobstorage/client.go b/internal/component/azure/blobstorage/client.go index de32f9bde..cfd76ab57 100644 --- a/internal/component/azure/blobstorage/client.go +++ b/internal/component/azure/blobstorage/client.go @@ -83,7 +83,7 @@ func CreateContainerStorageClient(log logger.Logger, meta map[string]string) (*c } client, clientErr = container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) if clientErr != nil { - return nil, nil, fmt.Errorf("cannot init Blobstorage container client: %w", err) + return nil, nil, fmt.Errorf("cannot init Blobstorage container client: %w", clientErr) } } else { // fallback to AAD diff --git a/pubsub/azure/eventhubs/entity_management.go b/pubsub/azure/eventhubs/entity_management.go index 5cd226f44..6fd25bfb3 100644 --- a/pubsub/azure/eventhubs/entity_management.go +++ b/pubsub/azure/eventhubs/entity_management.go @@ -46,7 +46,7 @@ const ( ) // Intializes the entity management capabilities. This method is invoked by Init. -func (aeh *AzureEventHubs) initEntityManagement(md map[string]string) error { +func (aeh *AzureEventHubs) initEntityManagement() error { // Validate the metadata err := aeh.validateEnitityManagementMetadata() if err != nil { @@ -54,7 +54,7 @@ func (aeh *AzureEventHubs) initEntityManagement(md map[string]string) error { } // Get Azure Management plane credentials object - settings, err := azauth.NewEnvironmentSettings("azure", md) + settings, err := azauth.NewEnvironmentSettings("azure", aeh.metadata.properties) if err != nil { return err } diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 088cc3ab4..f2a70e85e 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -98,7 +98,7 @@ func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { aeh.logger.Info("connecting to Azure Event Hub using Azure AD; the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored") if aeh.metadata.EnableEntityManagement { - err = aeh.initEntityManagement(metadata.Properties) + err = aeh.initEntityManagement() if err != nil { return fmt.Errorf("failed to initialize entity manager: %w", err) } @@ -237,6 +237,24 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su return fmt.Errorf("error trying to establish a connection: %w", err) } + // Ensure that no subscriber using the old "track 1" SDK is active + // TODO(@ItalyPaleAle): Remove this for Dapr 1.13 + { + ctx, cancel := context.WithTimeout(subscribeCtx, 10*time.Minute) + err = aeh.ensureNoTrack1Subscribers(ctx, req.Topic) + cancel() + if err != nil { + // If there's a timeout, it means that the other client was still active after the timeout + // In this case, we actually panic to make sure the user notices the error and ensures the rollout of the new version of Dapr is complete + if errors.Is(err, context.DeadlineExceeded) { + aeh.logger.Fatalf("Another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr, and this is not supported. Please ensure that all applications subscribed to the same topic, with this consumer group, are using Dapr 1.10 or newer.") + } + + // In case of other errors, just return the error + return fmt.Errorf("failed to check for subscribers using an old version of Dapr") + } + } + // This component has built-in retries because Event Hubs doesn't support N/ACK for messages retryHandler := func(ctx context.Context, msg *pubsub.NewMessage) error { b := aeh.backOffConfig.NewBackOffWithContext(subscribeCtx) diff --git a/pubsub/azure/eventhubs/metadata.go b/pubsub/azure/eventhubs/metadata.go index fa3ded2c7..78d7dfa10 100644 --- a/pubsub/azure/eventhubs/metadata.go +++ b/pubsub/azure/eventhubs/metadata.go @@ -41,6 +41,7 @@ type azureEventHubsMetadata struct { // Internal properties hubName string aadTokenProvider azcore.TokenCredential + properties map[string]string } func parseEventHubsMetadata(meta pubsub.Metadata, log logger.Logger) (*azureEventHubsMetadata, error) { @@ -50,6 +51,9 @@ func parseEventHubsMetadata(meta pubsub.Metadata, log logger.Logger) (*azureEven return nil, fmt.Errorf("failed to decode metada: %w", err) } + // Store the raw properties in the object + m.properties = meta.Properties + // One and only one of connectionString and eventHubNamespace is required if m.ConnectionString == "" && m.EventHubNamespace == "" { return nil, errors.New("one of connectionString or eventHubNamespace is required") diff --git a/pubsub/azure/eventhubs/track1_upgrade.go b/pubsub/azure/eventhubs/track1_upgrade.go new file mode 100644 index 000000000..ebae8db1e --- /dev/null +++ b/pubsub/azure/eventhubs/track1_upgrade.go @@ -0,0 +1,112 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eventhubs + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + + azauth "github.com/dapr/components-contrib/internal/authentication/azure" + "github.com/dapr/kit/logger" +) + +// This method ensures that there are currently no active subscribers to the same Event Hub topic that are using the old ("track 1") SDK of Azure Event Hubs. This is the SDK that was in use until Dapr 1.9. +// Because the new SDK stores checkpoints in a different way, clients using the new ("track 2") and the old SDK cannot coexist. +// To ensure this doesn't happen, when we create a new subscription to the same topic and with the same consumer group, we check if there's a file in Azure Storage with the checkpoint created by the old SDK and with a still-active lease. If that's true, we wait up to 10 minutes before we crash Dapr with a log message describing what's happening. +// These conflicts should be transient anyways, as mixed versions of Dapr should only happen during a rollout of a new version of Dapr. +// TODO(@ItalyPaleAle): Remove this for Dapr 1.13 +func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, topic string) error { + // Get a client to Azure Blob Storage + client, err := aeh.createContainerStorageClient() + if err != nil { + return err + } + + // In the old version of the SDK, checkpoints were stored in the root of the storage account and were named like: + // `dapr-(topic)-(consumer-group)-(partition-key)` + // We need to list those up and check if they have an active lease + prefix := fmt.Sprintf("dapr-%s-%s-", topic, aeh.metadata.ConsumerGroup) + pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Prefix: &prefix, + }) + for pager.More() { + ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) + resp, err := pager.NextPage(ctx) + cancel() + if err != nil { + fmt.Errorf("failed to list blobs: %w", err) + } + for _, blob := range resp.Segment.BlobItems { + fmt.Println(*blob.Name) + } + } + + return nil +} + +func (aeh *AzureEventHubs) createContainerStorageClient() (*container.Client, error) { + options := container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Telemetry: policy.TelemetryOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + }, + }, + } + + var ( + err error + client *container.Client + ) + // Use the global URL for Azure Storage + containerURL := fmt.Sprintf("https://%s.blob.%s/%s", aeh.metadata.StorageAccountName, "core.windows.net", aeh.metadata.StorageContainerName) + + if aeh.metadata.StorageConnectionString != "" { + // Authenticate with a connection string + client, err = container.NewClientFromConnectionString(aeh.metadata.StorageConnectionString, aeh.metadata.StorageContainerName, &options) + if err != nil { + return nil, fmt.Errorf("error creating Azure Storage client from connection string: %w", err) + } + } else if aeh.metadata.StorageAccountKey != "" { + // Authenticate with a shared key + credential, newSharedKeyErr := azblob.NewSharedKeyCredential(aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey) + if newSharedKeyErr != nil { + return nil, fmt.Errorf("invalid Azure Storage shared key credentials with error: %w", newSharedKeyErr) + } + client, err = container.NewClientWithSharedKeyCredential(containerURL, credential, &options) + if err != nil { + return nil, fmt.Errorf("error creating Azure Storage client from shared key credentials: %w", err) + } + } else { + // Use Azure AD + settings, err := azauth.NewEnvironmentSettings("storage", aeh.metadata.properties) + if err != nil { + return nil, err + } + credential, tokenErr := settings.GetTokenCredential() + if tokenErr != nil { + return nil, fmt.Errorf("invalid Azure Storage token credentials with error: %w", tokenErr) + } + client, err = container.NewClient(containerURL, credential, &options) + if err != nil { + return nil, fmt.Errorf("error creating Azure Storage client from token credentials: %w", err) + } + } + + return client, nil +} diff --git a/tests/certification/bindings/azure/blobstorage/go.mod b/tests/certification/bindings/azure/blobstorage/go.mod index 9599955e3..d05e8426f 100644 --- a/tests/certification/bindings/azure/blobstorage/go.mod +++ b/tests/certification/bindings/azure/blobstorage/go.mod @@ -17,7 +17,7 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect diff --git a/tests/certification/bindings/azure/blobstorage/go.sum b/tests/certification/bindings/azure/blobstorage/go.sum index acb0ff902..aceab4e19 100644 --- a/tests/certification/bindings/azure/blobstorage/go.sum +++ b/tests/certification/bindings/azure/blobstorage/go.sum @@ -40,8 +40,8 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= diff --git a/tests/certification/bindings/azure/cosmosdb/go.mod b/tests/certification/bindings/azure/cosmosdb/go.mod index 9da6a627e..3bde702fd 100644 --- a/tests/certification/bindings/azure/cosmosdb/go.mod +++ b/tests/certification/bindings/azure/cosmosdb/go.mod @@ -18,8 +18,8 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go v67.0.0+incompatible // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect diff --git a/tests/certification/bindings/azure/cosmosdb/go.sum b/tests/certification/bindings/azure/cosmosdb/go.sum index 6c6e5f9d6..8804a0c49 100644 --- a/tests/certification/bindings/azure/cosmosdb/go.sum +++ b/tests/certification/bindings/azure/cosmosdb/go.sum @@ -40,10 +40,10 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v67.0.0+incompatible h1:SVBwznSETB0Sipd0uyGJr7khLhJOFRUEUb+0JgkCvDo= -github.com/Azure/azure-sdk-for-go v67.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3 h1:x1shk+tVZ6kLwIQMn4r+pdz8szo3mA0jd8STmgh+aRk= diff --git a/tests/certification/bindings/azure/eventhubs/go.mod b/tests/certification/bindings/azure/eventhubs/go.mod index 51d4a28be..17769b4a4 100644 --- a/tests/certification/bindings/azure/eventhubs/go.mod +++ b/tests/certification/bindings/azure/eventhubs/go.mod @@ -19,8 +19,8 @@ require ( github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-event-hubs-go/v3 v3.4.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go v67.0.0+incompatible // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect diff --git a/tests/certification/bindings/azure/eventhubs/go.sum b/tests/certification/bindings/azure/eventhubs/go.sum index 43f4754fe..3795c4e7b 100644 --- a/tests/certification/bindings/azure/eventhubs/go.sum +++ b/tests/certification/bindings/azure/eventhubs/go.sum @@ -42,10 +42,10 @@ github.com/Azure/azure-event-hubs-go/v3 v3.4.0/go.mod h1:ODgt5C1/c73FToYj+mWojUJ github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v67.0.0+incompatible h1:SVBwznSETB0Sipd0uyGJr7khLhJOFRUEUb+0JgkCvDo= -github.com/Azure/azure-sdk-for-go v67.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= diff --git a/tests/certification/bindings/azure/servicebusqueues/go.mod b/tests/certification/bindings/azure/servicebusqueues/go.mod index c3904ab2c..d4518fc39 100644 --- a/tests/certification/bindings/azure/servicebusqueues/go.mod +++ b/tests/certification/bindings/azure/servicebusqueues/go.mod @@ -17,7 +17,7 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4 // indirect diff --git a/tests/certification/bindings/azure/servicebusqueues/go.sum b/tests/certification/bindings/azure/servicebusqueues/go.sum index 16c73b156..b9d4b2c06 100644 --- a/tests/certification/bindings/azure/servicebusqueues/go.sum +++ b/tests/certification/bindings/azure/servicebusqueues/go.sum @@ -40,8 +40,8 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= diff --git a/tests/certification/bindings/azure/storagequeues/go.mod b/tests/certification/bindings/azure/storagequeues/go.mod index fedac58be..422b576eb 100644 --- a/tests/certification/bindings/azure/storagequeues/go.mod +++ b/tests/certification/bindings/azure/storagequeues/go.mod @@ -17,7 +17,7 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect diff --git a/tests/certification/bindings/azure/storagequeues/go.sum b/tests/certification/bindings/azure/storagequeues/go.sum index 52cb97e8b..f448eb77d 100644 --- a/tests/certification/bindings/azure/storagequeues/go.sum +++ b/tests/certification/bindings/azure/storagequeues/go.sum @@ -40,8 +40,8 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= diff --git a/tests/certification/pubsub/azure/eventhubs/go.mod b/tests/certification/pubsub/azure/eventhubs/go.mod index b43089b17..74dd78093 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.mod +++ b/tests/certification/pubsub/azure/eventhubs/go.mod @@ -18,11 +18,12 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 // indirect github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/pubsub/azure/eventhubs/go.sum b/tests/certification/pubsub/azure/eventhubs/go.sum index 6a957384b..a27a814f2 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.sum +++ b/tests/certification/pubsub/azure/eventhubs/go.sum @@ -40,8 +40,8 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= @@ -50,6 +50,8 @@ github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 h1:X/ePaAG8gu github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0/go.mod h1:5dog28UP3dd1BnCPFYvyHfsmA+Phmoezt+KWT5cZnyc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 h1:BWeAAEzkCnL0ABVJqs+4mYudNch7oFGPtTlSmIWL8ms= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0/go.mod h1:Y3gnVwfaz8h6L1YHar+NfWORtBoVUSB5h4GlGkdeF7Q= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/pubsub/azure/servicebus/topics/go.mod b/tests/certification/pubsub/azure/servicebus/topics/go.mod index 8f48c72c4..88e9de803 100644 --- a/tests/certification/pubsub/azure/servicebus/topics/go.mod +++ b/tests/certification/pubsub/azure/servicebus/topics/go.mod @@ -18,7 +18,7 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4 // indirect diff --git a/tests/certification/pubsub/azure/servicebus/topics/go.sum b/tests/certification/pubsub/azure/servicebus/topics/go.sum index 16c73b156..b9d4b2c06 100644 --- a/tests/certification/pubsub/azure/servicebus/topics/go.sum +++ b/tests/certification/pubsub/azure/servicebus/topics/go.sum @@ -40,8 +40,8 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= diff --git a/tests/certification/secretstores/azure/keyvault/go.mod b/tests/certification/secretstores/azure/keyvault/go.mod index fdcae16c0..8c7eaf6ab 100644 --- a/tests/certification/secretstores/azure/keyvault/go.mod +++ b/tests/certification/secretstores/azure/keyvault/go.mod @@ -16,7 +16,7 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 // indirect diff --git a/tests/certification/secretstores/azure/keyvault/go.sum b/tests/certification/secretstores/azure/keyvault/go.sum index e1b69214d..75326a28c 100644 --- a/tests/certification/secretstores/azure/keyvault/go.sum +++ b/tests/certification/secretstores/azure/keyvault/go.sum @@ -40,8 +40,8 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= diff --git a/tests/certification/state/azure/blobstorage/go.mod b/tests/certification/state/azure/blobstorage/go.mod index b76bb4094..cc146030b 100644 --- a/tests/certification/state/azure/blobstorage/go.mod +++ b/tests/certification/state/azure/blobstorage/go.mod @@ -16,7 +16,7 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 // indirect diff --git a/tests/certification/state/azure/blobstorage/go.sum b/tests/certification/state/azure/blobstorage/go.sum index acb0ff902..aceab4e19 100644 --- a/tests/certification/state/azure/blobstorage/go.sum +++ b/tests/certification/state/azure/blobstorage/go.sum @@ -40,8 +40,8 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= diff --git a/tests/certification/state/azure/cosmosdb/go.mod b/tests/certification/state/azure/cosmosdb/go.mod index 8968400f3..3b94d6670 100644 --- a/tests/certification/state/azure/cosmosdb/go.mod +++ b/tests/certification/state/azure/cosmosdb/go.mod @@ -16,8 +16,8 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go v67.0.0+incompatible // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect diff --git a/tests/certification/state/azure/cosmosdb/go.sum b/tests/certification/state/azure/cosmosdb/go.sum index cd64770c9..c0deadffe 100644 --- a/tests/certification/state/azure/cosmosdb/go.sum +++ b/tests/certification/state/azure/cosmosdb/go.sum @@ -40,10 +40,10 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v67.0.0+incompatible h1:SVBwznSETB0Sipd0uyGJr7khLhJOFRUEUb+0JgkCvDo= -github.com/Azure/azure-sdk-for-go v67.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3 h1:x1shk+tVZ6kLwIQMn4r+pdz8szo3mA0jd8STmgh+aRk= diff --git a/tests/certification/state/azure/tablestorage/go.mod b/tests/certification/state/azure/tablestorage/go.mod index 95e2d1338..e44425664 100644 --- a/tests/certification/state/azure/tablestorage/go.mod +++ b/tests/certification/state/azure/tablestorage/go.mod @@ -16,7 +16,7 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect diff --git a/tests/certification/state/azure/tablestorage/go.sum b/tests/certification/state/azure/tablestorage/go.sum index c605fced5..c18655f36 100644 --- a/tests/certification/state/azure/tablestorage/go.sum +++ b/tests/certification/state/azure/tablestorage/go.sum @@ -40,8 +40,8 @@ github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8 github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgDAUZ+I2bO8J7s8JxXmu9fhi2ss= From fcfcbe9d8710d99a547c2bd0ff997399c5911718 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:46:38 +0000 Subject: [PATCH 16/42] Handle smooth upgrade from Track 1 SDKs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (If you consider crashing Dapr on purpose "smooth" 🤣) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 13 ++++-- pubsub/azure/eventhubs/track1_upgrade.go | 59 +++++++++++++++++------- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index f2a70e85e..6ae4c4300 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -240,14 +240,21 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su // Ensure that no subscriber using the old "track 1" SDK is active // TODO(@ItalyPaleAle): Remove this for Dapr 1.13 { - ctx, cancel := context.WithTimeout(subscribeCtx, 10*time.Minute) + ctx, cancel := context.WithTimeout(subscribeCtx, 2*time.Minute) err = aeh.ensureNoTrack1Subscribers(ctx, req.Topic) cancel() if err != nil { // If there's a timeout, it means that the other client was still active after the timeout - // In this case, we actually panic to make sure the user notices the error and ensures the rollout of the new version of Dapr is complete + // In this case, we return an error here so Dapr can continue the initialization and report a "healthy" status (but this subscription won't be active) + // After 2 minutes, then, we panic, which ensures that during a rollout Kubernetes will see that this pod is unhealthy and re-creates that. Hopefully, by then other instances of the app will have been updated and no more locks will be present if errors.Is(err, context.DeadlineExceeded) { - aeh.logger.Fatalf("Another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr, and this is not supported. Please ensure that all applications subscribed to the same topic, with this consumer group, are using Dapr 1.10 or newer.") + errMsg := fmt.Sprintf("Another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr, and this is not supported. Please ensure that all applications subscribed to the same topic, with this consumer group, are using Dapr 1.10 or newer.", req.Topic) + aeh.logger.Error(errMsg + " ⚠️⚠️⚠️ Dapr will crash in 2 minutes to force the orchestrator to restart the process after the rollout of other instances is complete.") + go func() { + time.Sleep(2 * time.Minute) + aeh.logger.Fatalf("Another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr, and this is not supported. Please ensure that all applications subscribed to the same topic, with this consumer group, are using Dapr 1.10 or newer.", req.Topic) + }() + return fmt.Errorf("another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr", req.Topic) } // In case of other errors, just return the error diff --git a/pubsub/azure/eventhubs/track1_upgrade.go b/pubsub/azure/eventhubs/track1_upgrade.go index ebae8db1e..3cbb1f72c 100644 --- a/pubsub/azure/eventhubs/track1_upgrade.go +++ b/pubsub/azure/eventhubs/track1_upgrade.go @@ -15,20 +15,24 @@ package eventhubs import ( "context" + "errors" "fmt" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/cenkalti/backoff/v4" azauth "github.com/dapr/components-contrib/internal/authentication/azure" "github.com/dapr/kit/logger" + "github.com/dapr/kit/retry" ) // This method ensures that there are currently no active subscribers to the same Event Hub topic that are using the old ("track 1") SDK of Azure Event Hubs. This is the SDK that was in use until Dapr 1.9. // Because the new SDK stores checkpoints in a different way, clients using the new ("track 2") and the old SDK cannot coexist. -// To ensure this doesn't happen, when we create a new subscription to the same topic and with the same consumer group, we check if there's a file in Azure Storage with the checkpoint created by the old SDK and with a still-active lease. If that's true, we wait up to 10 minutes before we crash Dapr with a log message describing what's happening. +// To ensure this doesn't happen, when we create a new subscription to the same topic and with the same consumer group, we check if there's a file in Azure Storage with the checkpoint created by the old SDK and with a still-active lease. If that's true, we wait (until the context expires) before we crash Dapr with a log message describing what's happening. // These conflicts should be transient anyways, as mixed versions of Dapr should only happen during a rollout of a new version of Dapr. // TODO(@ItalyPaleAle): Remove this for Dapr 1.13 func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, topic string) error { @@ -42,22 +46,45 @@ func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, // `dapr-(topic)-(consumer-group)-(partition-key)` // We need to list those up and check if they have an active lease prefix := fmt.Sprintf("dapr-%s-%s-", topic, aeh.metadata.ConsumerGroup) - pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ - Prefix: &prefix, - }) - for pager.More() { - ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) - resp, err := pager.NextPage(ctx) - cancel() - if err != nil { - fmt.Errorf("failed to list blobs: %w", err) - } - for _, blob := range resp.Segment.BlobItems { - fmt.Println(*blob.Name) - } - } - return nil + // Retry until we find no active leases - or the context expires + backOffConfig := retry.DefaultConfig() + backOffConfig.Policy = retry.PolicyExponential + backOffConfig.MaxInterval = time.Minute + backOffConfig.MaxElapsedTime = 0 + backOffConfig.MaxRetries = -1 + b := backOffConfig.NewBackOffWithContext(parentCtx) + err = backoff.Retry(func() error { + pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Prefix: &prefix, + }) + for pager.More() { + ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) + resp, err := pager.NextPage(ctx) + cancel() + if err != nil { + return fmt.Errorf("failed to list blobs: %w", err) + } + for _, blob := range resp.Segment.BlobItems { + if blob == nil || blob.Name == nil || blob.Properties == nil || blob.Properties.LeaseState == nil { + continue + } + aeh.logger.Debugf("Found checkpoint from an older Dapr version %s", *blob.Name) + // If the blob is locked, it means that there's another Dapr process with an old version of the SDK running, so we need to wait + if *blob.Properties.LeaseStatus == "locked" { + aeh.logger.Warnf("Found active lease on checkpoint %s from an older Dapr version - waiting for lease to expire", *blob.Name) + return fmt.Errorf("found active lease on checkpoint %s from an old Dapr version", *blob.Name) + } + } + } + return nil + }, b) + + // If the error is a DeadlineExceeded on the operation and not on parentCtx, handle that separately to avoid crashing Dapr needlessly + if err != nil && errors.Is(err, context.DeadlineExceeded) && parentCtx.Err() != context.DeadlineExceeded { + err = errors.New("failed to list blobs: request timed out") + } + return err } func (aeh *AzureEventHubs) createContainerStorageClient() (*container.Client, error) { From 2c98c8ef259ec6d0dc17dc84a68663359f218052 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:50:05 +0000 Subject: [PATCH 17/42] =?UTF-8?q?=F0=9F=92=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- pubsub/azure/eventhubs/eventhubs.go | 3 ++- pubsub/azure/eventhubs/track1_upgrade.go | 6 +++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 3d4c22b85..33d1acf27 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( dubbo.apache.org/dubbo-go/v3 v3.0.3-0.20230118042253-4f159a2b38f3 github.com/Azure/azure-amqp-common-go/v4 v4.0.0 github.com/Azure/azure-event-hubs-go/v3 v3.4.0 - github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + github.com/Azure/azure-sdk-for-go v65.0.0+incompatible github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0 diff --git a/go.sum b/go.sum index e69528199..5b2490708 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/Azure/azure-event-hubs-go/v3 v3.4.0/go.mod h1:ODgt5C1/c73FToYj+mWojUJ github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= +github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 6ae4c4300..12fb9fd8c 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -33,6 +33,7 @@ import ( "github.com/dapr/components-contrib/pubsub" "github.com/dapr/components-contrib/pubsub/azure/eventhubs/conn" "github.com/dapr/kit/logger" + "github.com/dapr/kit/ptr" "github.com/dapr/kit/retry" ) @@ -167,7 +168,7 @@ func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPubl Body: entry.Event, } if entry.ContentType != "" { - messages[i].ContentType = &entry.ContentType + messages[i].ContentType = ptr.Of(entry.ContentType) } if val := entry.Metadata["partitionKey"]; val != "" { if batchOpts.PartitionKey != nil && *batchOpts.PartitionKey != val { diff --git a/pubsub/azure/eventhubs/track1_upgrade.go b/pubsub/azure/eventhubs/track1_upgrade.go index 3cbb1f72c..ba9dc417c 100644 --- a/pubsub/azure/eventhubs/track1_upgrade.go +++ b/pubsub/azure/eventhubs/track1_upgrade.go @@ -60,10 +60,10 @@ func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, }) for pager.More() { ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) - resp, err := pager.NextPage(ctx) + resp, innerErr := pager.NextPage(ctx) cancel() - if err != nil { - return fmt.Errorf("failed to list blobs: %w", err) + if innerErr != nil { + return fmt.Errorf("failed to list blobs: %w", innerErr) } for _, blob := range resp.Segment.BlobItems { if blob == nil || blob.Name == nil || blob.Properties == nil || blob.Properties.LeaseState == nil { From 7153e75549077dcaaec1b65e9485bb0746254aa2 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:53:38 +0000 Subject: [PATCH 18/42] Do not.... crash twice Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 12fb9fd8c..7625b0957 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -19,6 +19,7 @@ import ( "fmt" "strconv" "sync" + "sync/atomic" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -49,6 +50,9 @@ type AzureEventHubs struct { checkpointStoreLock *sync.RWMutex managementCreds azcore.TokenCredential + + // TODO(@ItalyPaleAle): Remove in Dapr 1.13 + isFailed *atomic.Bool } // NewAzureEventHubs returns a new Azure Event hubs instance. @@ -58,6 +62,7 @@ func NewAzureEventHubs(logger logger.Logger) pubsub.PubSub { producersLock: &sync.RWMutex{}, producers: make(map[string]*azeventhubs.ProducerClient, 1), checkpointStoreLock: &sync.RWMutex{}, + isFailed: &atomic.Bool{}, } } @@ -241,6 +246,11 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su // Ensure that no subscriber using the old "track 1" SDK is active // TODO(@ItalyPaleAle): Remove this for Dapr 1.13 { + // If a previous topic already failed, no need to try with other topics, as we're about to panic anyways + if aeh.isFailed.Load() { + return errors.New("subscribing to another topic on this component failed and Dapr is scheduled to crash; will not try subscribing to a new topic") + } + ctx, cancel := context.WithTimeout(subscribeCtx, 2*time.Minute) err = aeh.ensureNoTrack1Subscribers(ctx, req.Topic) cancel() @@ -249,6 +259,7 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su // In this case, we return an error here so Dapr can continue the initialization and report a "healthy" status (but this subscription won't be active) // After 2 minutes, then, we panic, which ensures that during a rollout Kubernetes will see that this pod is unhealthy and re-creates that. Hopefully, by then other instances of the app will have been updated and no more locks will be present if errors.Is(err, context.DeadlineExceeded) { + aeh.isFailed.Store(true) errMsg := fmt.Sprintf("Another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr, and this is not supported. Please ensure that all applications subscribed to the same topic, with this consumer group, are using Dapr 1.10 or newer.", req.Topic) aeh.logger.Error(errMsg + " ⚠️⚠️⚠️ Dapr will crash in 2 minutes to force the orchestrator to restart the process after the rollout of other instances is complete.") go func() { From 462d7a95be5c59790a22c568474217f565b4a036 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 18:10:40 +0000 Subject: [PATCH 19/42] Remove old file Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go.old | 715 ------------------------ 1 file changed, 715 deletions(-) delete mode 100644 pubsub/azure/eventhubs/eventhubs.go.old diff --git a/pubsub/azure/eventhubs/eventhubs.go.old b/pubsub/azure/eventhubs/eventhubs.go.old deleted file mode 100644 index 0c50b0758..000000000 --- a/pubsub/azure/eventhubs/eventhubs.go.old +++ /dev/null @@ -1,715 +0,0 @@ -/* -Copyright 2021 The Dapr Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package eventhubs - -import ( - "context" - "errors" - "fmt" - "strconv" - "strings" - "time" - - "github.com/Azure/azure-amqp-common-go/v4/aad" - "github.com/Azure/azure-amqp-common-go/v4/conn" - eventhub "github.com/Azure/azure-event-hubs-go/v3" - "github.com/Azure/azure-event-hubs-go/v3/eph" - "github.com/Azure/azure-event-hubs-go/v3/storage" - mgmt "github.com/Azure/azure-sdk-for-go/services/eventhub/mgmt/2017-04-01/eventhub" - "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/Azure/go-autorest/autorest/azure" - - azauth "github.com/dapr/components-contrib/internal/authentication/azure" - "github.com/dapr/components-contrib/internal/utils" - "github.com/dapr/components-contrib/metadata" - contribMetadata "github.com/dapr/components-contrib/metadata" - "github.com/dapr/components-contrib/pubsub" - "github.com/dapr/kit/logger" - "github.com/dapr/kit/retry" -) - -const ( - // connection string entity path key. - entityPathKey = "EntityPath" - // metadata partitionKey key. - partitionKeyMetadataKey = "partitionKey" - - // errors. - hubManagerCreationErrorMsg = "failed to create eventHub manager client" - invalidConnectionStringErrorMsg = "connectionString is invalid" - missingConnectionStringNamespaceErrorMsg = "connectionString or eventHubNamespace is required" - missingStorageAccountNameErrorMsg = "storageAccountName is a required attribute for subscribe" - missingStorageAccountKeyErrorMsg = "storageAccountKey is required for subscribe when connectionString is provided" - missingStorageContainerNameErrorMsg = "storageContainerName is a required attribute for subscribe" - missingConsumerIDErrorMsg = "missing consumerID attribute for subscribe" - bothConnectionStringNamespaceErrorMsg = "both connectionString and eventHubNamespace are given, only one should be given" - missingResourceGroupNameMsg = "missing resourceGroupName attribute required for entityManagement" - missingSubscriptionIDMsg = "missing subscriptionID attribute required for entityManagement" - entityManagementConnectionStrMsg = "entity management support is not available with connectionString" - differentTopicConnectionStringErrorTmpl = "specified topic %s does not match the Event Hub name in the provided connectionString" - - // Event Hubs SystemProperties names for metadata passthrough. - sysPropSequenceNumber = "x-opt-sequence-number" - sysPropEnqueuedTime = "x-opt-enqueued-time" - sysPropOffset = "x-opt-offset" - sysPropPartitionID = "x-opt-partition-id" - sysPropPartitionKey = "x-opt-partition-key" - sysPropIotHubDeviceConnectionID = "iothub-connection-device-id" - sysPropIotHubAuthGenerationID = "iothub-connection-auth-generation-id" - sysPropIotHubConnectionAuthMethod = "iothub-connection-auth-method" - sysPropIotHubConnectionModuleID = "iothub-connection-module-id" - sysPropIotHubEnqueuedTime = "iothub-enqueuedtime" - sysPropMessageID = "message-id" - - // Metadata field to ensure all Event Hub properties pass through - requireAllProperties = "requireAllProperties" - - defaultMessageRetentionInDays = 1 - defaultPartitionCount = 1 - - resourceCheckMaxRetry = 5 - resourceCheckMaxRetryInterval time.Duration = 5 * time.Minute - resourceCreationTimeout time.Duration = 15 * time.Second - resourceGetTimeout time.Duration = 5 * time.Second - - // See https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-quotas for numbers. - maxMessageRetention int32 = 90 - maxPartitionCount int32 = 1024 -) - -func subscribeHandler(ctx context.Context, topic string, getAllProperties bool, e *eventhub.Event, handler pubsub.Handler) error { - res := pubsub.NewMessage{Data: e.Data, Topic: topic, Metadata: map[string]string{}} - if e.SystemProperties.SequenceNumber != nil { - res.Metadata[sysPropSequenceNumber] = strconv.FormatInt(*e.SystemProperties.SequenceNumber, 10) - } - if e.SystemProperties.EnqueuedTime != nil { - res.Metadata[sysPropEnqueuedTime] = e.SystemProperties.EnqueuedTime.Format(time.RFC3339) - } - if e.SystemProperties.Offset != nil { - res.Metadata[sysPropOffset] = strconv.FormatInt(*e.SystemProperties.Offset, 10) - } - // According to azure-event-hubs-go docs, this will always be nil. - if e.SystemProperties.PartitionID != nil { - res.Metadata[sysPropPartitionID] = strconv.Itoa(int(*e.SystemProperties.PartitionID)) - } - // The following metadata properties are only present if event was generated by Azure IoT Hub. - if e.SystemProperties.PartitionKey != nil { - res.Metadata[sysPropPartitionKey] = *e.SystemProperties.PartitionKey - } - if e.SystemProperties.IoTHubDeviceConnectionID != nil { - res.Metadata[sysPropIotHubDeviceConnectionID] = *e.SystemProperties.IoTHubDeviceConnectionID - } - if e.SystemProperties.IoTHubAuthGenerationID != nil { - res.Metadata[sysPropIotHubAuthGenerationID] = *e.SystemProperties.IoTHubAuthGenerationID - } - if e.SystemProperties.IoTHubConnectionAuthMethod != nil { - res.Metadata[sysPropIotHubConnectionAuthMethod] = *e.SystemProperties.IoTHubConnectionAuthMethod - } - if e.SystemProperties.IoTHubConnectionModuleID != nil { - res.Metadata[sysPropIotHubConnectionModuleID] = *e.SystemProperties.IoTHubConnectionModuleID - } - if e.SystemProperties.IoTHubEnqueuedTime != nil { - res.Metadata[sysPropIotHubEnqueuedTime] = e.SystemProperties.IoTHubEnqueuedTime.Format(time.RFC3339) - } - // azure-event-hubs-go SDK pulls out the AMQP message-id property to the Event.ID property, map it from there. - if e.ID != "" { - res.Metadata[sysPropMessageID] = e.ID - } - // added properties if any ( includes application properties from iot-hub) - if getAllProperties { - if e.Properties != nil && len(e.Properties) > 0 { - for key, value := range e.Properties { - if str, ok := value.(string); ok { - res.Metadata[key] = str - } - } - } - } - - return handler(ctx, &res) -} - -// AzureEventHubs allows sending/receiving Azure Event Hubs events. -type AzureEventHubs struct { - metadata *azureEventHubsMetadata - logger logger.Logger - backOffConfig retry.Config - hubClients map[string]*eventhub.Hub - eventProcessors map[string]*eph.EventProcessorHost - hubManager *eventhub.HubManager - eventHubSettings azauth.EnvironmentSettings - managementSettings azauth.EnvironmentSettings - cgClient *mgmt.ConsumerGroupsClient - tokenProvider *aad.TokenProvider - storageCredential azblob.Credential - azureEnvironment *azure.Environment -} - -type azureEventHubsMetadata struct { - ConnectionString string `json:"connectionString,omitempty" mapstructure:"connectionString"` - EventHubNamespace string `json:"eventHubNamespace,omitempty" mapstructure:"eventHubNamespace"` - ConsumerGroup string `json:"consumerID" mapstructure:"consumerID"` - StorageAccountName string `json:"storageAccountName,omitempty" mapstructure:"storageAccountName"` - StorageAccountKey string `json:"storageAccountKey,omitempty" mapstructure:"storageAccountKey"` - StorageContainerName string `json:"storageContainerName,omitempty" mapstructure:"storageContainerName"` - EnableEntityManagement bool `json:"enableEntityManagement,omitempty,string" mapstructure:"enableEntityManagement"` - MessageRetentionInDays int32 `json:"messageRetentionInDays,omitempty,string" mapstructure:"messageRetentionInDays"` - PartitionCount int32 `json:"partitionCount,omitempty,string" mapstructure:"partitionCount"` - SubscriptionID string `json:"subscriptionID,omitempty" mapstructure:"subscriptionID"` - ResourceGroupName string `json:"resourceGroupName,omitempty" mapstructure:"resourceGroupName"` -} - -// NewAzureEventHubs returns a new Azure Event hubs instance. -func NewAzureEventHubs(logger logger.Logger) pubsub.PubSub { - return &AzureEventHubs{logger: logger} -} - -func parseEventHubsMetadata(meta pubsub.Metadata) (*azureEventHubsMetadata, error) { - var m azureEventHubsMetadata - err := metadata.DecodeMetadata(meta.Properties, &m) - if err != nil { - return nil, fmt.Errorf("failed to decode metada: %w", err) - } - - if m.ConnectionString == "" && m.EventHubNamespace == "" { - return nil, errors.New(missingConnectionStringNamespaceErrorMsg) - } - - if m.ConnectionString != "" && m.EventHubNamespace != "" { - return nil, errors.New(bothConnectionStringNamespaceErrorMsg) - } - - return &m, nil -} - -func validateAndGetHubName(connectionString string) (string, error) { - parsed, err := conn.ParsedConnectionFromStr(connectionString) - if err != nil { - return "", err - } - return parsed.HubName, nil -} - -func (aeh *AzureEventHubs) ensureEventHub(ctx context.Context, hubName string) error { - if aeh.hubManager == nil { - return fmt.Errorf("hubManager client not initialized properly") - } - entity, err := aeh.getHubEntity(ctx, hubName) - if err != nil { - return err - } - if entity == nil { - if err := aeh.createHubEntity(ctx, hubName); err != nil { - return err - } - } - return nil -} - -func (aeh *AzureEventHubs) ensureSubscription(ctx context.Context, hubName string) error { - err := aeh.ensureEventHub(ctx, hubName) - if err != nil { - return err - } - _, err = aeh.getConsumerGroupsClient() - if err != nil { - return err - } - return aeh.createConsumerGroup(ctx, hubName) -} - -func (aeh *AzureEventHubs) getConsumerGroupsClient() (*mgmt.ConsumerGroupsClient, error) { - if aeh.cgClient != nil { - return aeh.cgClient, nil - } - client := mgmt.NewConsumerGroupsClientWithBaseURI(aeh.managementSettings.AzureEnvironment.ResourceManagerEndpoint, - aeh.metadata.SubscriptionID) - a, err := aeh.managementSettings.GetAuthorizer() - if err != nil { - return nil, err - } - client.Authorizer = a - aeh.cgClient = &client - return aeh.cgClient, nil -} - -func (aeh *AzureEventHubs) createConsumerGroup(parentCtx context.Context, hubName string) error { - create := false - backOffConfig := retry.DefaultConfig() - backOffConfig.Policy = retry.PolicyExponential - backOffConfig.MaxInterval = resourceCheckMaxRetryInterval - backOffConfig.MaxRetries = resourceCheckMaxRetry - - b := backOffConfig.NewBackOffWithContext(parentCtx) - - err := retry.NotifyRecover(func() error { - c, err := aeh.shouldCreateConsumerGroup(parentCtx, hubName) - if err == nil { - create = c - return nil - } - return err - }, b, func(_ error, _ time.Duration) { - aeh.logger.Errorf("Error checking for consumer group for EventHub : %s. Retrying...", hubName) - }, func() { - aeh.logger.Warnf("Successfully checked for consumer group in EventHub %s after it previously failed.", hubName) - }) - if err != nil { - return err - } - if create { - ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) - _, err = aeh.cgClient.CreateOrUpdate(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup, mgmt.ConsumerGroup{}) - cancel() - if err != nil { - return err - } - } - return nil -} - -func (aeh *AzureEventHubs) shouldCreateConsumerGroup(parentCtx context.Context, hubName string) (bool, error) { - ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) - g, err := aeh.cgClient.Get(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup) - cancel() - if err != nil { - if g.HasHTTPStatus(404) { - return true, nil - } - return false, err - } - if *g.Name == aeh.metadata.ConsumerGroup { - aeh.logger.Infof("consumer group %s exists for the requested topic/eventHub %s", aeh.metadata.ConsumerGroup, hubName) - } - return false, nil -} - -func (aeh *AzureEventHubs) getHubEntity(parentCtx context.Context, hubName string) (*eventhub.HubEntity, error) { - ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) - defer cancel() - return aeh.hubManager.Get(ctx, hubName) -} - -func (aeh *AzureEventHubs) createHubEntity(parentCtx context.Context, hubName string) error { - ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) - _, err := aeh.hubManager.Put(ctx, hubName, - eventhub.HubWithMessageRetentionInDays(aeh.metadata.MessageRetentionInDays), - eventhub.HubWithPartitionCount(aeh.metadata.PartitionCount)) - cancel() - if err != nil { - aeh.logger.Errorf("error creating event hub %s: %s", hubName, err) - return fmt.Errorf("error creating event hub %s: %s", hubName, err) - } - return nil -} - -func (aeh *AzureEventHubs) ensurePublisherClient(ctx context.Context, hubName string) error { - if aeh.metadata.EnableEntityManagement { - if err := aeh.ensureEventHub(ctx, hubName); err != nil { - return err - } - } - userAgent := "dapr-" + logger.DaprVersion - if aeh.metadata.ConnectionString != "" { - // Connect with connection string. - newConnectionString, err := aeh.constructConnectionStringFromTopic(hubName) - if err != nil { - return err - } - - hub, err := eventhub.NewHubFromConnectionString(newConnectionString, - eventhub.HubWithUserAgent(userAgent)) - if err != nil { - aeh.logger.Debugf("unable to connect to azure event hubs: %v", err) - return fmt.Errorf("unable to connect to azure event hubs: %v", err) - } - aeh.hubClients[hubName] = hub - } else { - if hubName == "" { - return errors.New("error: missing topic/hubName attribute with AAD connection") - } - - hub, err := eventhub.NewHub(aeh.metadata.EventHubNamespace, hubName, aeh.tokenProvider, eventhub.HubWithUserAgent(userAgent)) - if err != nil { - return fmt.Errorf("unable to connect to azure event hubs: %v", err) - } - aeh.hubClients[hubName] = hub - } - - return nil -} - -func (aeh *AzureEventHubs) ensureSubscriberClient(ctx context.Context, topic string, leaserCheckpointer *storage.LeaserCheckpointer) (*eph.EventProcessorHost, error) { - // connectionString given. - if aeh.metadata.ConnectionString != "" { - hubName, err := validateAndGetHubName(aeh.metadata.ConnectionString) - if err != nil { - return nil, fmt.Errorf("error parsing connection string %s", err) - } - if hubName != "" && hubName != topic { - return nil, fmt.Errorf("error: component cannot subscribe to requested topic %s with the given connectionString", topic) - } - if hubName == "" { - aeh.logger.Debugf("eventhub namespace connection string given. using topic as event hub entity path") - } - connectionString, err := aeh.constructConnectionStringFromTopic(topic) - if err != nil { - return nil, err - } - processor, err := eph.NewFromConnectionString( - ctx, connectionString, - leaserCheckpointer, - leaserCheckpointer, - eph.WithNoBanner(), - eph.WithConsumerGroup(aeh.metadata.ConsumerGroup), - ) - if err != nil { - return nil, err - } - aeh.logger.Debugf("processor initialized via connection string for topic %s", topic) - return processor, nil - } - // AAD connection. - processor, err := eph.New(ctx, - aeh.metadata.EventHubNamespace, - topic, - aeh.tokenProvider, - leaserCheckpointer, - leaserCheckpointer, - eph.WithNoBanner(), - eph.WithConsumerGroup(aeh.metadata.ConsumerGroup), - ) - if err != nil { - return nil, err - } - aeh.logger.Debugf("processor initialized via AAD for topic %s", topic) - - return processor, nil -} - -func (aeh *AzureEventHubs) createHubManager() error { - // Only AAD based authentication supported. - hubManager, err := eventhub.NewHubManagerFromAzureEnvironment(aeh.metadata.EventHubNamespace, aeh.tokenProvider, *aeh.eventHubSettings.AzureEnvironment) - if err != nil { - return fmt.Errorf("%s %s", hubManagerCreationErrorMsg, err) - } - aeh.hubManager = hubManager - - return nil -} - -func (aeh *AzureEventHubs) constructConnectionStringFromTopic(requestedTopic string) (string, error) { - hubName, err := validateAndGetHubName(aeh.metadata.ConnectionString) - if err != nil { - return "", err - } - if hubName != "" && hubName == requestedTopic { - return aeh.metadata.ConnectionString, nil - } else if hubName != "" { - return "", fmt.Errorf(differentTopicConnectionStringErrorTmpl, requestedTopic) - } - return aeh.metadata.ConnectionString + ";" + entityPathKey + "=" + requestedTopic, nil -} - -func (aeh *AzureEventHubs) validateEnitityManagementMetadata() error { - if aeh.metadata.MessageRetentionInDays <= 0 || aeh.metadata.MessageRetentionInDays > maxMessageRetention { - aeh.logger.Warnf("invalid/no message retention time period is given with entity management enabled, default value of %d is used", defaultMessageRetentionInDays) - aeh.metadata.MessageRetentionInDays = defaultMessageRetentionInDays - } - if aeh.metadata.PartitionCount <= 0 || aeh.metadata.PartitionCount > maxPartitionCount { - aeh.logger.Warnf("invalid/no partition count is given with entity management enabled, default value of %d is used", defaultPartitionCount) - aeh.metadata.PartitionCount = defaultPartitionCount - } - if aeh.metadata.ResourceGroupName == "" { - return errors.New(missingResourceGroupNameMsg) - } - if aeh.metadata.SubscriptionID == "" { - return errors.New(missingSubscriptionIDMsg) - } - return nil -} - -func (aeh *AzureEventHubs) validateSubscriptionAttributes() error { - m := *aeh.metadata - - if m.StorageAccountName == "" { - return errors.New("storageAccountName is a required attribute for subscribe") - } - - if m.StorageAccountKey == "" && m.ConnectionString != "" { - return errors.New("storageAccountKey is required for subscribe when connectionString is provided") - } - - if m.StorageContainerName == "" { - return errors.New("storageContainerName is a required attribute for subscribe") - } - - if m.ConsumerGroup == "" { - return errors.New("consumerID is a required attribute for subscribe") - } - return nil -} - -func (aeh *AzureEventHubs) getStoragePrefixString(topic string) string { - return fmt.Sprintf("dapr-%s-%s-", topic, aeh.metadata.ConsumerGroup) -} - -// Init connects to Azure Event Hubs. -func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { - m, parseErr := parseEventHubsMetadata(metadata) - if parseErr != nil { - return parseErr - } - - aeh.metadata = m - aeh.eventProcessors = map[string]*eph.EventProcessorHost{} - aeh.hubClients = map[string]*eventhub.Hub{} - - if aeh.metadata.ConnectionString != "" { - // Validate connectionString. - hubName, validateErr := validateAndGetHubName(aeh.metadata.ConnectionString) - if validateErr != nil { - return errors.New(invalidConnectionStringErrorMsg) - } - if hubName != "" { - aeh.logger.Infof("connectionString provided is specific to event hub %q. Publishing or subscribing to a topic that does not match this event hub will fail when attempted.", hubName) - } else { - aeh.logger.Infof("hubName not given in connectionString. connection established on first publish/subscribe") - aeh.logger.Debugf("req.Topic field in incoming requests honored") - } - if aeh.metadata.EnableEntityManagement { - // See https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-management-libraries - return errors.New(entityManagementConnectionStrMsg) - } - } else { - // Connect via AAD. - settings, sErr := azauth.NewEnvironmentSettings(azauth.AzureEventHubsResourceName, metadata.Properties) - if sErr != nil { - return sErr - } - aeh.eventHubSettings = settings - tokenProvider, err := aeh.eventHubSettings.GetAMQPTokenProvider() - if err != nil { - return fmt.Errorf("%s %s", hubManagerCreationErrorMsg, err) - } - aeh.tokenProvider = tokenProvider - aeh.logger.Info("connecting to Azure EventHubs via AAD. connection established on first publish/subscribe") - aeh.logger.Debugf("req.Topic field in incoming requests honored") - - if aeh.metadata.EnableEntityManagement { - if err := aeh.validateEnitityManagementMetadata(); err != nil { - return err - } - - // Create hubManager for eventHub management with AAD. - if managerCreateErr := aeh.createHubManager(); managerCreateErr != nil { - return managerCreateErr - } - - // Get Azure Management plane settings for creating consumer groups using event hubs management client. - settings, err := azauth.NewEnvironmentSettings("azure", metadata.Properties) - if err != nil { - return err - } - aeh.managementSettings = settings - } - } - - // connect to the storage account. - if m.StorageAccountKey != "" { - metadata.Properties["accountKey"] = m.StorageAccountKey - } - var storageCredsErr error - aeh.storageCredential, aeh.azureEnvironment, storageCredsErr = azauth.GetAzureStorageBlobCredentials(aeh.logger, m.StorageAccountName, metadata.Properties) - if storageCredsErr != nil { - return fmt.Errorf("invalid storage credentials with error: %w", storageCredsErr) - } - - // Default retry configuration is used if no backOff properties are set. - if err := retry.DecodeConfigWithPrefix( - &aeh.backOffConfig, - metadata.Properties, - "backOff"); err != nil { - return err - } - - return nil -} - -// Publish sends data to Azure Event Hubs. -func (aeh *AzureEventHubs) Publish(ctx context.Context, req *pubsub.PublishRequest) error { - if _, ok := aeh.hubClients[req.Topic]; !ok { - if err := aeh.ensurePublisherClient(ctx, req.Topic); err != nil { - return fmt.Errorf("error on establishing hub connection: %s", err) - } - } - event := &eventhub.Event{Data: req.Data} - val, ok := req.Metadata[partitionKeyMetadataKey] - if ok { - event.PartitionKey = &val - } - err := aeh.hubClients[req.Topic].Send(ctx, event) - if err != nil { - return fmt.Errorf("error from publish: %s", err) - } - - return nil -} - -// BulkPublish sends data to Azure Event Hubs in bulk. -func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPublishRequest) (pubsub.BulkPublishResponse, error) { - if _, ok := aeh.hubClients[req.Topic]; !ok { - if err := aeh.ensurePublisherClient(ctx, req.Topic); err != nil { - err = fmt.Errorf("error on establishing hub connection: %s", err) - return pubsub.NewBulkPublishResponse(req.Entries, err), err - } - } - - // Create a slice of events to send. - events := make([]*eventhub.Event, len(req.Entries)) - for i, entry := range req.Entries { - events[i] = &eventhub.Event{Data: entry.Event} - if val, ok := entry.Metadata[partitionKeyMetadataKey]; ok { - events[i].PartitionKey = &val - } - } - - // Configure options for sending events. - opts := []eventhub.BatchOption{ - eventhub.BatchWithMaxSizeInBytes(utils.GetElemOrDefaultFromMap( - req.Metadata, contribMetadata.MaxBulkPubBytesKey, int(eventhub.DefaultMaxMessageSizeInBytes))), - } - - // Send events. - err := aeh.hubClients[req.Topic].SendBatch(ctx, eventhub.NewEventBatchIterator(events...), opts...) - if err != nil { - // Partial success is not supported by Azure Event Hubs. - // If an error occurs, all events are considered failed. - return pubsub.NewBulkPublishResponse(req.Entries, err), err - } - - return pubsub.BulkPublishResponse{}, nil -} - -// Subscribe receives data from Azure Event Hubs. -func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) error { - err := aeh.validateSubscriptionAttributes() - if err != nil { - return fmt.Errorf("error : error on subscribe %s", err) - } - if aeh.metadata.EnableEntityManagement { - if err = aeh.ensureSubscription(subscribeCtx, req.Topic); err != nil { - return err - } - } - - // Set topic name, consumerID prefix for partition checkpoint lease blob path. - // This is needed to support multiple consumers for the topic using the same storage container. - leaserPrefixOpt := storage.WithPrefixInBlobPath(aeh.getStoragePrefixString(req.Topic)) - leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(aeh.storageCredential, aeh.metadata.StorageAccountName, aeh.metadata.StorageContainerName, *aeh.azureEnvironment, leaserPrefixOpt) - if err != nil { - return err - } - - processor, err := aeh.ensureSubscriberClient(subscribeCtx, req.Topic, leaserCheckpointer) - if err != nil { - return err - } - - getAllProperties := false - if req.Metadata[requireAllProperties] != "" { - getAllProperties, err = strconv.ParseBool(req.Metadata[requireAllProperties]) - if err != nil { - aeh.logger.Errorf("invalid value for metadata : %s . Error: %v.", requireAllProperties, err) - } - } - - aeh.logger.Debugf("registering handler for topic %s", req.Topic) - _, err = processor.RegisterHandler(subscribeCtx, - func(_ context.Context, e *eventhub.Event) error { - // This component has built-in retries because Event Hubs doesn't support N/ACK for messages - b := aeh.backOffConfig.NewBackOffWithContext(subscribeCtx) - - retryerr := retry.NotifyRecover(func() error { - aeh.logger.Debugf("Processing EventHubs event %s/%s", req.Topic, e.ID) - - return subscribeHandler(subscribeCtx, req.Topic, getAllProperties, e, handler) - }, b, func(_ error, _ time.Duration) { - aeh.logger.Warnf("Error processing EventHubs event: %s/%s. Retrying...", req.Topic, e.ID) - }, func() { - aeh.logger.Warnf("Successfully processed EventHubs event after it previously failed: %s/%s", req.Topic, e.ID) - }) - if retryerr != nil { - aeh.logger.Errorf("Too many failed attempts at processing Eventhubs event: %s/%s. Error: %v.", req.Topic, e.ID, err) - } - return retryerr - }) - if err != nil { - return err - } - - err = processor.StartNonBlocking(subscribeCtx) - if err != nil { - return err - } - aeh.eventProcessors[req.Topic] = processor - - // Listen for context cancelation and stop processing messages - // This seems to be necessary because otherwise the processor isn't automatically closed on context cancelation - go func() { - <-subscribeCtx.Done() - stopCtx, stopCancel := context.WithTimeout(context.Background(), resourceGetTimeout) - stopErr := processor.Close(stopCtx) - stopCancel() - if stopErr != nil { - aeh.logger.Warnf("Error closing subscribe processor: %v", stopErr) - } - }() - - return nil -} - -func (aeh *AzureEventHubs) Close() (err error) { - flag := false - var ctx context.Context - var cancel context.CancelFunc - for topic, client := range aeh.hubClients { - ctx, cancel = context.WithTimeout(context.Background(), resourceGetTimeout) - err = client.Close(ctx) - cancel() - if err != nil { - flag = true - aeh.logger.Warnf("error closing publish client properly for topic/eventHub %s: %s", topic, err) - } - } - aeh.hubClients = map[string]*eventhub.Hub{} - for topic, client := range aeh.eventProcessors { - ctx, cancel = context.WithTimeout(context.Background(), resourceGetTimeout) - err = client.Close(ctx) - cancel() - if err != nil { - flag = true - aeh.logger.Warnf("error closing event processor host client properly for topic/eventHub %s: %s", topic, err) - } - } - aeh.eventProcessors = map[string]*eph.EventProcessorHost{} - if flag { - return errors.New("error closing event hub clients in a proper fashion") - } - return nil -} - -func (aeh *AzureEventHubs) Features() []pubsub.Feature { - return nil -} From 0649d5fae9ef5d66b741bc27ba9c58e0413f2799 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 19:00:15 +0000 Subject: [PATCH 20/42] Updated tests Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../azure/eventhubs/eventhubs_integration_test.go | 15 ++++++++------- pubsub/azure/eventhubs/track1_upgrade.go | 3 ++- tests/scripts/send-iot-device-events.sh | 0 3 files changed, 10 insertions(+), 8 deletions(-) mode change 100644 => 100755 tests/scripts/send-iot-device-events.sh diff --git a/pubsub/azure/eventhubs/eventhubs_integration_test.go b/pubsub/azure/eventhubs/eventhubs_integration_test.go index d9699365a..0df138a27 100644 --- a/pubsub/azure/eventhubs/eventhubs_integration_test.go +++ b/pubsub/azure/eventhubs/eventhubs_integration_test.go @@ -28,7 +28,7 @@ import ( "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" - "github.com/dapr/kit/logger" + kitLogger "github.com/dapr/kit/logger" ) const ( @@ -36,7 +36,7 @@ const ( // iotHubConnectionStringEnvKey defines the key containing the integration test connection string // For the default EventHub endpoint for an Azure IoT Hub, it will resemble: - // Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=service;SharedAccessKey=;EntityPath= + // Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=service;SharedAccessKey=;EntityPath=integration-test-topic iotHubConnectionStringEnvKey = "AzureIotHubEventHubConnectionString" iotHubConsumerGroupEnvKey = "AzureIotHubPubsubConsumerGroup" iotHubNameEnvKey = "AzureIotHubName" @@ -66,17 +66,18 @@ func createIotHubPubsubMetadata() pubsub.Metadata { } func testReadIotHubEvents(t *testing.T) { - logger := logger.NewLogger("pubsub.azure.eventhubs.integration.test") + logger := kitLogger.NewLogger("pubsub.azure.eventhubs.integration.test") + logger.SetOutputLevel(kitLogger.DebugLevel) eh := NewAzureEventHubs(logger).(*AzureEventHubs) err := eh.Init(createIotHubPubsubMetadata()) - assert.Nil(t, err) + assert.NoError(t, err) // Invoke az CLI via bash script to send test IoT device events // Requires the AZURE_CREDENTIALS environment variable to be already set (output of `az ad sp create-for-rbac`) cmd := exec.Command("/bin/bash", "../../../tests/scripts/send-iot-device-events.sh") cmd.Env = append(os.Environ(), fmt.Sprintf("IOT_HUB_NAME=%s", os.Getenv(iotHubNameEnvKey))) out, err := cmd.CombinedOutput() - assert.Nil(t, err, "Error in send-iot-device-events.sh:\n%s", out) + assert.NoError(t, err, "Error in send-iot-device-events.sh:\n%s", string(out)) // Setup subscription to capture messages in a closure so that test asserts can be // performed on the main thread, including the case where the handler is never invoked. @@ -87,13 +88,13 @@ func testReadIotHubEvents(t *testing.T) { } req := pubsub.SubscribeRequest{ - Topic: testTopic, // TODO: Handle Topic configuration after EventHubs pubsub rewrite #951 + Topic: testTopic, Metadata: map[string]string{ "requireAllProperties": "true", }, } err = eh.Subscribe(context.Background(), req, handler) - assert.Nil(t, err) + assert.NoError(t, err) // Note: azure-event-hubs-go SDK defaultLeasePersistenceInterval is 5s // Sleep long enough so that the azure event hubs SDK has time to persist updated checkpoints diff --git a/pubsub/azure/eventhubs/track1_upgrade.go b/pubsub/azure/eventhubs/track1_upgrade.go index ba9dc417c..3e66196db 100644 --- a/pubsub/azure/eventhubs/track1_upgrade.go +++ b/pubsub/azure/eventhubs/track1_upgrade.go @@ -63,7 +63,8 @@ func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, resp, innerErr := pager.NextPage(ctx) cancel() if innerErr != nil { - return fmt.Errorf("failed to list blobs: %w", innerErr) + // Treat these errors as permanent + return backoff.Permanent(fmt.Errorf("failed to list blobs: %w", innerErr)) } for _, blob := range resp.Segment.BlobItems { if blob == nil || blob.Name == nil || blob.Properties == nil || blob.Properties.LeaseState == nil { diff --git a/tests/scripts/send-iot-device-events.sh b/tests/scripts/send-iot-device-events.sh old mode 100644 new mode 100755 From c83919a49d2883f20bbd9c30bf0a12a48d7cf494 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 19:34:01 +0000 Subject: [PATCH 21/42] Removed vendored conn library Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/conn/README.md | 8 --- pubsub/azure/eventhubs/conn/conn.go | 92 ------------------------ pubsub/azure/eventhubs/eventhubs.go | 14 ++-- pubsub/azure/eventhubs/eventhubs_test.go | 12 +--- pubsub/azure/eventhubs/metadata.go | 13 ++++ 5 files changed, 18 insertions(+), 121 deletions(-) delete mode 100644 pubsub/azure/eventhubs/conn/README.md delete mode 100644 pubsub/azure/eventhubs/conn/conn.go diff --git a/pubsub/azure/eventhubs/conn/README.md b/pubsub/azure/eventhubs/conn/README.md deleted file mode 100644 index b131a6a0a..000000000 --- a/pubsub/azure/eventhubs/conn/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# conn - -This package is copied from the Azure SDK as it is an internal package in the upstream SDK. - -We can remove this once [Azure/azure-sdk-for-go#19840](https://github.com/Azure/azure-sdk-for-go/issues/19840) is fixed. - -- Source: https://github.com/Azure/azure-sdk-for-go/tree/sdk/messaging/azeventhubs/v0.4.0/sdk/messaging/azeventhubs/internal/conn -- License: Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. diff --git a/pubsub/azure/eventhubs/conn/conn.go b/pubsub/azure/eventhubs/conn/conn.go deleted file mode 100644 index 54f23fba2..000000000 --- a/pubsub/azure/eventhubs/conn/conn.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package conn - -import ( - "errors" - "fmt" - "net/url" - "strings" -) - -type ( - // ParsedConn is the structure of a parsed Service Bus or Event Hub connection string. - ParsedConn struct { - Namespace string - HubName string - - KeyName string - Key string - - SAS string - } -) - -// ParsedConnectionFromStr takes a string connection string from the Azure portal and returns the parsed representation. -// The method will return an error if the Endpoint, SharedAccessKeyName or SharedAccessKey is empty. -func ParsedConnectionFromStr(connStr string) (*ParsedConn, error) { - const ( - endpointKey = "Endpoint" - sharedAccessKeyNameKey = "SharedAccessKeyName" - sharedAccessKeyKey = "SharedAccessKey" - entityPathKey = "EntityPath" - sharedAccessSignatureKey = "SharedAccessSignature" - ) - - // We can parse two types of connection strings. - // 1. Connection strings generated from the portal (or elsewhere) that contain an embedded key and keyname. - // 2. A specially formatted connection string with an embedded SharedAccessSignature: - // Endpoint=sb://.servicebus.windows.net;SharedAccessSignature=SharedAccessSignature sr=.servicebus.windows.net&sig=&se=&skn=" - var namespace, hubName, keyName, secret, sas string - splits := strings.Split(connStr, ";") - - for _, split := range splits { - keyAndValue := strings.SplitN(split, "=", 2) - if len(keyAndValue) < 2 { - return nil, errors.New("failed parsing connection string due to unmatched key value separated by '='") - } - - // if a key value pair has `=` in the value, recombine them - key := keyAndValue[0] - value := strings.Join(keyAndValue[1:], "=") - switch { - case strings.EqualFold(endpointKey, key): - u, err := url.Parse(value) - if err != nil { - return nil, errors.New("failed parsing connection string due to an incorrectly formatted Endpoint value") - } - namespace = u.Host - case strings.EqualFold(sharedAccessKeyNameKey, key): - keyName = value - case strings.EqualFold(sharedAccessKeyKey, key): - secret = value - case strings.EqualFold(entityPathKey, key): - hubName = value - case strings.EqualFold(sharedAccessSignatureKey, key): - sas = value - } - } - - parsed := &ParsedConn{ - Namespace: namespace, - KeyName: keyName, - Key: secret, - HubName: hubName, - SAS: sas, - } - - if namespace == "" { - return parsed, fmt.Errorf("key %q must not be empty", endpointKey) - } - - if sas == "" && keyName == "" { - return parsed, fmt.Errorf("key %q must not be empty", sharedAccessKeyNameKey) - } - - if secret == "" && sas == "" { - return parsed, fmt.Errorf("key %q or %q cannot both be empty", sharedAccessKeyKey, sharedAccessSignatureKey) - } - - return parsed, nil -} diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 7625b0957..e2a65cf3d 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -32,7 +32,6 @@ import ( "github.com/dapr/components-contrib/internal/utils" contribMetadata "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" - "github.com/dapr/components-contrib/pubsub/azure/eventhubs/conn" "github.com/dapr/kit/logger" "github.com/dapr/kit/ptr" "github.com/dapr/kit/retry" @@ -76,19 +75,14 @@ func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { if aeh.metadata.ConnectionString != "" { // Connect using the connection string - var parsedConn *conn.ParsedConn - parsedConn, err = conn.ParsedConnectionFromStr(aeh.metadata.ConnectionString) - if err != nil { - return fmt.Errorf("connectionString is invalid: %w", err) - } - - if parsedConn.HubName != "" { - aeh.logger.Infof(`The provided connection string is specific to the Event Hub ("entity path") '%s'; publishing or subscribing to a topic that does not match this Event Hub will fail when attempted`, parsedConn.HubName) + hubName := hubNameFromConnString(aeh.metadata.ConnectionString) + if hubName != "" { + aeh.logger.Infof(`The provided connection string is specific to the Event Hub ("entity path") '%s'; publishing or subscribing to a topic that does not match this Event Hub will fail when attempted`, hubName) } else { aeh.logger.Infof(`The provided connection string does not contain an Event Hub name ("entity path"); the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored`) } - aeh.metadata.hubName = parsedConn.HubName + aeh.metadata.hubName = hubName } else { // Connect via Azure AD var env azauth.EnvironmentSettings diff --git a/pubsub/azure/eventhubs/eventhubs_test.go b/pubsub/azure/eventhubs/eventhubs_test.go index 2f300d3e3..621fc8a50 100644 --- a/pubsub/azure/eventhubs/eventhubs_test.go +++ b/pubsub/azure/eventhubs/eventhubs_test.go @@ -89,6 +89,7 @@ func TestConstructConnectionStringFromTopic(t *testing.T) { require.NoError(t, err) assert.Equal(t, connectionString+";EntityPath=testHub", c) }) + t.Run("valid connectionString with hub name", func(t *testing.T) { connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key;EntityPath=testHub" topic := "testHub" @@ -106,18 +107,7 @@ func TestConstructConnectionStringFromTopic(t *testing.T) { require.NoError(t, err) assert.Equal(t, connectionString, c) }) - t.Run("invalid connectionString with hub name", func(t *testing.T) { - connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;NoKey=key;EntityPath=testHub" - metadata := pubsub.Metadata{Base: metadata.Base{ - Properties: map[string]string{ - "connectionString": connectionString, - }, - }} - aeh := &AzureEventHubs{logger: testLogger} - err := aeh.Init(metadata) - require.Error(t, err) - }) t.Run("valid connectionString with different hub name", func(t *testing.T) { connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key;EntityPath=testHub" topic := "differentHub" diff --git a/pubsub/azure/eventhubs/metadata.go b/pubsub/azure/eventhubs/metadata.go index 78d7dfa10..2c55b7db2 100644 --- a/pubsub/azure/eventhubs/metadata.go +++ b/pubsub/azure/eventhubs/metadata.go @@ -16,6 +16,7 @@ package eventhubs import ( "errors" "fmt" + "regexp" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -75,3 +76,15 @@ func parseEventHubsMetadata(meta pubsub.Metadata, log logger.Logger) (*azureEven return &m, nil } + +var hubNameMatch = regexp.MustCompile(`(?i)(^|;)EntityPath=([^;]+)(;|$)`) + +// Returns the hub name (topic) from the connection string. +// TODO: Temporary until https://github.com/Azure/azure-sdk-for-go/issues/19840 is fixed - then use `conn.ParsedConnectionFromStr(aeh.metadata.ConnectionString)` and look at the `HubName` property. +func hubNameFromConnString(connString string) string { + match := hubNameMatch.FindStringSubmatch(connString) + if len(match) < 3 { + return "" + } + return match[2] +} From d97dac3b984d6e9cd48acb8efccd1691d7fd6003 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 20:37:59 +0000 Subject: [PATCH 22/42] Fixes for Azure AD auth Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/entity_management.go | 20 ++++++++++---------- pubsub/azure/eventhubs/eventhubs.go | 2 +- pubsub/azure/eventhubs/metadata.go | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pubsub/azure/eventhubs/entity_management.go b/pubsub/azure/eventhubs/entity_management.go index 6fd25bfb3..16550b25e 100644 --- a/pubsub/azure/eventhubs/entity_management.go +++ b/pubsub/azure/eventhubs/entity_management.go @@ -98,15 +98,15 @@ func (aeh *AzureEventHubs) ensureEventHubEntity(parentCtx context.Context, topic return fmt.Errorf("failed to create Event Hubs ARM client: %w", err) } - aeh.logger.Debugf("Checking if entity %s exists on Event Hub namespace %s", topic, aeh.metadata.EventHubNamespace) + aeh.logger.Debugf("Checking if entity %s exists on Event Hub namespace %s", topic, aeh.metadata.namespaceName) // Check if the entity exists ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) defer cancel() - _, err = client.Get(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, topic, nil) + _, err = client.Get(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.namespaceName, topic, nil) if err == nil { // If there's no error, the entity already exists, so all good - aeh.logger.Debugf("Entity %s already exists on Event Hub namespace %s", topic, aeh.metadata.EventHubNamespace) + aeh.logger.Debugf("Entity %s already exists on Event Hub namespace %s", topic, aeh.metadata.namespaceName) return nil } @@ -118,7 +118,7 @@ func (aeh *AzureEventHubs) ensureEventHubEntity(parentCtx context.Context, topic } // Create the entity - aeh.logger.Infof("Will create entity %s on Event Hub namespace %s", topic, aeh.metadata.EventHubNamespace) + aeh.logger.Infof("Will create entity %s on Event Hub namespace %s", topic, aeh.metadata.namespaceName) params := armeventhub.Eventhub{ Properties: &armeventhub.Properties{ MessageRetentionInDays: ptr.Of(int64(aeh.metadata.MessageRetentionInDays)), @@ -127,12 +127,12 @@ func (aeh *AzureEventHubs) ensureEventHubEntity(parentCtx context.Context, topic } ctx, cancel = context.WithTimeout(parentCtx, resourceCreationTimeout) defer cancel() - _, err = client.CreateOrUpdate(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, topic, params, nil) + _, err = client.CreateOrUpdate(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.namespaceName, topic, params, nil) if err != nil { return fmt.Errorf("failed to create Event Hub entity %s: %w", topic, err) } - aeh.logger.Infof("Entity %s created on Event Hub namespace %s", topic, aeh.metadata.EventHubNamespace) + aeh.logger.Infof("Entity %s created on Event Hub namespace %s", topic, aeh.metadata.namespaceName) return nil } @@ -184,7 +184,7 @@ func (aeh *AzureEventHubs) ensureSubscription(parentCtx context.Context, hubName ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) defer cancel() - _, err = client.CreateOrUpdate(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup, armeventhub.ConsumerGroup{}, nil) + _, err = client.CreateOrUpdate(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.namespaceName, hubName, aeh.metadata.ConsumerGroup, armeventhub.ConsumerGroup{}, nil) if err != nil { return fmt.Errorf("failed to create consumer group %s: %w", aeh.metadata.ConsumerGroup, err) } @@ -196,10 +196,10 @@ func (aeh *AzureEventHubs) ensureSubscription(parentCtx context.Context, hubName func (aeh *AzureEventHubs) shouldCreateConsumerGroup(parentCtx context.Context, client *armeventhub.ConsumerGroupsClient, hubName string) (bool, error) { ctx, cancel := context.WithTimeout(parentCtx, resourceGetTimeout) defer cancel() - _, err := client.Get(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.EventHubNamespace, hubName, aeh.metadata.ConsumerGroup, nil) + _, err := client.Get(ctx, aeh.metadata.ResourceGroupName, aeh.metadata.namespaceName, hubName, aeh.metadata.ConsumerGroup, nil) if err == nil { // If there's no error, the consumer group already exists, so all good - return true, nil + return false, nil } // Check if the error is NotFound or something else @@ -210,5 +210,5 @@ func (aeh *AzureEventHubs) shouldCreateConsumerGroup(parentCtx context.Context, } // Consumer group doesn't exist - return false, nil + return true, nil } diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index e2a65cf3d..312de4907 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -264,7 +264,7 @@ func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.Su } // In case of other errors, just return the error - return fmt.Errorf("failed to check for subscribers using an old version of Dapr") + return fmt.Errorf("failed to check for subscribers using an old version of Dapr: %w", err) } } diff --git a/pubsub/azure/eventhubs/metadata.go b/pubsub/azure/eventhubs/metadata.go index 2c55b7db2..17fcd6566 100644 --- a/pubsub/azure/eventhubs/metadata.go +++ b/pubsub/azure/eventhubs/metadata.go @@ -17,6 +17,7 @@ import ( "errors" "fmt" "regexp" + "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -40,6 +41,7 @@ type azureEventHubsMetadata struct { ResourceGroupName string `json:"resourceGroupName" mapstructure:"resourceGroupName"` // Internal properties + namespaceName string hubName string aadTokenProvider azcore.TokenCredential properties map[string]string @@ -74,6 +76,18 @@ func parseEventHubsMetadata(meta pubsub.Metadata, log logger.Logger) (*azureEven log.Warn("Entity management support is not available when connecting with a connection string") } + if m.EventHubNamespace != "" { + // Older versions of Dapr required the namespace name to be just the name and not a FQDN + // Automatically append ".servicebus.windows.net" to make them a FQDN if not present, but show a log + if !strings.ContainsRune(m.EventHubNamespace, '.') { + m.EventHubNamespace += ".servicebus.windows.net" + log.Info("Property eventHubNamespace is not a FQDN; the suffix '.servicebus.windows.net' will be added automatically") + } + + // The namespace name is the first part of the FQDN, until the first dot + m.namespaceName = m.EventHubNamespace[0:strings.IndexRune(m.EventHubNamespace, '.')] + } + return &m, nil } From 827c949c999759181e7541dd5112199b8fe73130 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 21:34:03 +0000 Subject: [PATCH 23/42] Address review comments Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 312de4907..ab68b7fbd 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -145,10 +145,11 @@ func (aeh *AzureEventHubs) Publish(ctx context.Context, req *pubsub.PublishReque // BulkPublish sends data to Azure Event Hubs in bulk. func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPublishRequest) (pubsub.BulkPublishResponse, error) { - res := pubsub.BulkPublishResponse{} + var err error if req.Topic == "" { - return res, errors.New("parameter 'topic' is required") + err = errors.New("parameter 'topic' is required") + return pubsub.NewBulkPublishResponse(req.Entries, err), err } // Batch options @@ -171,21 +172,22 @@ func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPubl } if val := entry.Metadata["partitionKey"]; val != "" { if batchOpts.PartitionKey != nil && *batchOpts.PartitionKey != val { - return res, errors.New("cannot send messages to different partitions") + err = errors.New("cannot send messages to different partitions") + return pubsub.NewBulkPublishResponse(req.Entries, err), err } batchOpts.PartitionKey = &val } } // Publish the message - err := aeh.doPublish(ctx, req.Topic, messages, batchOpts) + err = aeh.doPublish(ctx, req.Topic, messages, batchOpts) if err != nil { // Partial success is not supported by Azure Event Hubs. // If an error occurs, all events are considered failed. return pubsub.NewBulkPublishResponse(req.Entries, err), err } - return res, nil + return pubsub.BulkPublishResponse{}, nil } // Internal method used by Publish and BulkPublish to send messages @@ -199,7 +201,7 @@ func (aeh *AzureEventHubs) doPublish(ctx context.Context, topic string, messages // Build the batch of messages batch, err := client.NewEventDataBatch(ctx, batchOpts) if err != nil { - return fmt.Errorf("error creating batch: %w", err) + return fmt.Errorf("error creating event batch: %w", err) } // Add all messages @@ -357,7 +359,7 @@ func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, topic str for { // TODO: Support setting a batch size const batchSize = 1 - ctx, cancel = context.WithCancel(subscribeCtx) + ctx, cancel = context.WithTimeout(subscribeCtx, time.Minute) events, err = partitionClient.ReceiveEvents(ctx, batchSize, nil) cancel() @@ -367,7 +369,7 @@ func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, topic str // We'll just stop this subscription and return eventHubError := (*azeventhubs.Error)(nil) if errors.As(err, &eventHubError) && eventHubError.Code == azeventhubs.ErrorCodeOwnershipLost { - aeh.logger.Debug("Client lost ownership of partition %s for topic %s", partitionClient.PartitionID(), topic) + aeh.logger.Debugf("Client lost ownership of partition %s for topic %s", partitionClient.PartitionID(), topic) return nil } From c76526c80589ef9e5fb6776a70876b8a78946806 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 21:34:55 +0000 Subject: [PATCH 24/42] Fixed unit tests Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pubsub/azure/eventhubs/eventhubs_test.go b/pubsub/azure/eventhubs/eventhubs_test.go index 621fc8a50..2cb58d924 100644 --- a/pubsub/azure/eventhubs/eventhubs_test.go +++ b/pubsub/azure/eventhubs/eventhubs_test.go @@ -38,13 +38,23 @@ func TestParseEventHubsMetadata(t *testing.T) { }) t.Run("test namespace given", func(t *testing.T) { + props := map[string]string{"eventHubNamespace": "fake.servicebus.windows.net"} + + metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} + m, err := parseEventHubsMetadata(metadata, testLogger) + + require.NoError(t, err) + assert.Equal(t, "fake.servicebus.windows.net", m.EventHubNamespace) + }) + + t.Run("test namespace adds FQDN", func(t *testing.T) { props := map[string]string{"eventHubNamespace": "fake"} metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} m, err := parseEventHubsMetadata(metadata, testLogger) require.NoError(t, err) - assert.Equal(t, "fake", m.EventHubNamespace) + assert.Equal(t, "fake.servicebus.windows.net", m.EventHubNamespace) }) t.Run("test both connectionString and eventHubNamespace given", func(t *testing.T) { From a0251482c2f19b702effb8cb08b2c8eb6071792c Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 23:09:48 +0000 Subject: [PATCH 25/42] Create the storage container if it doesn't exist (to preserve compatibility with old SDK's behavior) Also fixes to tests Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 100 +++++++++++++++++- pubsub/azure/eventhubs/track1_upgrade.go | 66 ++---------- .../pubsub/azure/eventhubs/eventhubs_test.go | 56 +++++++--- 3 files changed, 144 insertions(+), 78 deletions(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index ab68b7fbd..85c705ee3 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -26,6 +26,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "golang.org/x/exp/maps" azauth "github.com/dapr/components-contrib/internal/authentication/azure" @@ -486,7 +488,7 @@ func (aeh *AzureEventHubs) getProducerClientForTopic(ctx context.Context, topic // Creates a processor for a given topic. func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic string) (*azeventhubs.Processor, error) { // Get the checkpoint store - checkpointStore, err := aeh.getCheckpointStore() + checkpointStore, err := aeh.getCheckpointStore(ctx) if err != nil { return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err) } @@ -546,7 +548,7 @@ func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic strin } // Returns the checkpoint store from the object. If it doesn't exist, it lazily initializes it. -func (aeh *AzureEventHubs) getCheckpointStore() (azeventhubs.CheckpointStore, error) { +func (aeh *AzureEventHubs) getCheckpointStore(ctx context.Context) (azeventhubs.CheckpointStore, error) { // Check if we have the checkpoint store aeh.checkpointStoreLock.RLock() if aeh.checkpointStoreCache != nil { @@ -565,7 +567,7 @@ func (aeh *AzureEventHubs) getCheckpointStore() (azeventhubs.CheckpointStore, er // Init the checkpoint store and store it in the object var err error - aeh.checkpointStoreCache, err = aeh.createCheckpointStore() + aeh.checkpointStoreCache, err = aeh.createCheckpointStore(ctx) if err != nil { return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err) } @@ -574,7 +576,7 @@ func (aeh *AzureEventHubs) getCheckpointStore() (azeventhubs.CheckpointStore, er } // Initializes a new checkpoint store -func (aeh *AzureEventHubs) createCheckpointStore() (checkpointStore azeventhubs.CheckpointStore, err error) { +func (aeh *AzureEventHubs) createCheckpointStore(ctx context.Context) (checkpointStore azeventhubs.CheckpointStore, err error) { if aeh.metadata.StorageAccountName == "" { return nil, errors.New("property storageAccountName is required to subscribe to an Event Hub topic") } @@ -582,6 +584,13 @@ func (aeh *AzureEventHubs) createCheckpointStore() (checkpointStore azeventhubs. return nil, errors.New("property storageContainerName is required to subscribe to an Event Hub topic") } + // Ensure the container exists + err = aeh.ensureStorageContainer(ctx) + if err != nil { + return nil, err + } + + // Create the checkpoint store checkpointStoreOpts := &checkpoints.BlobStoreOptions{ ClientOptions: policy.ClientOptions{ Telemetry: policy.TelemetryOptions{ @@ -620,6 +629,89 @@ func (aeh *AzureEventHubs) createCheckpointStore() (checkpointStore azeventhubs. return checkpointStore, nil } +// Ensures that the container exists in the Azure Storage Account. +// This is done to preserve backwards-compatibility with Dapr 1.9, as the old checkpoint SDK created them automatically. +func (aeh *AzureEventHubs) ensureStorageContainer(parentCtx context.Context) error { + // Get a client to Azure Blob Storage + client, err := aeh.createStorageClient() + if err != nil { + return err + } + + // Create the container + // This will return an error if it already exists + ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) + defer cancel() + _, err = client.CreateContainer(ctx, aeh.metadata.StorageContainerName, &container.CreateOptions{ + // Default is private + Access: nil, + }) + if err != nil { + // Check if it's an Azure Storage error + resErr := &azcore.ResponseError{} + // If the container already exists, return no error + if errors.As(err, &resErr) && (resErr.ErrorCode == "ContainerAlreadyExists" || resErr.ErrorCode == "ResourceAlreadyExists") { + return nil + } + + return fmt.Errorf("failed to create Azure Storage container %s: %w", aeh.metadata.StorageContainerName, err) + } + + return nil +} + +// Creates a client to access Azure Blob Storage +func (aeh *AzureEventHubs) createStorageClient() (*azblob.Client, error) { + options := azblob.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Telemetry: policy.TelemetryOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + }, + }, + } + + var ( + err error + client *azblob.Client + ) + // Use the global URL for Azure Storage + accountURL := fmt.Sprintf("https://%s.blob.%s", aeh.metadata.StorageAccountName, "core.windows.net") + + if aeh.metadata.StorageConnectionString != "" { + // Authenticate with a connection string + client, err = azblob.NewClientFromConnectionString(aeh.metadata.StorageConnectionString, &options) + if err != nil { + return nil, fmt.Errorf("error creating Azure Storage client from connection string: %w", err) + } + } else if aeh.metadata.StorageAccountKey != "" { + // Authenticate with a shared key + credential, newSharedKeyErr := azblob.NewSharedKeyCredential(aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey) + if newSharedKeyErr != nil { + return nil, fmt.Errorf("invalid Azure Storage shared key credentials with error: %w", newSharedKeyErr) + } + client, err = azblob.NewClientWithSharedKeyCredential(accountURL, credential, &options) + if err != nil { + return nil, fmt.Errorf("error creating Azure Storage client from shared key credentials: %w", err) + } + } else { + // Use Azure AD + settings, err := azauth.NewEnvironmentSettings("storage", aeh.metadata.properties) + if err != nil { + return nil, err + } + credential, tokenErr := settings.GetTokenCredential() + if tokenErr != nil { + return nil, fmt.Errorf("invalid Azure Storage token credentials with error: %w", tokenErr) + } + client, err = azblob.NewClient(accountURL, credential, &options) + if err != nil { + return nil, fmt.Errorf("error creating Azure Storage client from token credentials: %w", err) + } + } + + return client, nil +} + // Returns a connection string with the Event Hub name (entity path) set if not present. func (aeh *AzureEventHubs) constructConnectionStringFromTopic(topic string) (string, error) { if aeh.metadata.hubName != "" { diff --git a/pubsub/azure/eventhubs/track1_upgrade.go b/pubsub/azure/eventhubs/track1_upgrade.go index 3e66196db..72b901096 100644 --- a/pubsub/azure/eventhubs/track1_upgrade.go +++ b/pubsub/azure/eventhubs/track1_upgrade.go @@ -17,16 +17,13 @@ import ( "context" "errors" "fmt" + "net/http" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/cenkalti/backoff/v4" - azauth "github.com/dapr/components-contrib/internal/authentication/azure" - "github.com/dapr/kit/logger" "github.com/dapr/kit/retry" ) @@ -37,7 +34,7 @@ import ( // TODO(@ItalyPaleAle): Remove this for Dapr 1.13 func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, topic string) error { // Get a client to Azure Blob Storage - client, err := aeh.createContainerStorageClient() + client, err := aeh.createStorageClient() if err != nil { return err } @@ -55,7 +52,7 @@ func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, backOffConfig.MaxRetries = -1 b := backOffConfig.NewBackOffWithContext(parentCtx) err = backoff.Retry(func() error { - pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + pager := client.NewListBlobsFlatPager(aeh.metadata.StorageContainerName, &container.ListBlobsFlatOptions{ Prefix: &prefix, }) for pager.More() { @@ -64,6 +61,12 @@ func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, cancel() if innerErr != nil { // Treat these errors as permanent + resErr := &azcore.ResponseError{} + if !errors.As(err, &resErr) || resErr.StatusCode != http.StatusNotFound { + // A "not-found" error means that the storage container doesn't exist, so let's not handle it here + // Just return no error + return nil + } return backoff.Permanent(fmt.Errorf("failed to list blobs: %w", innerErr)) } for _, blob := range resp.Segment.BlobItems { @@ -87,54 +90,3 @@ func (aeh *AzureEventHubs) ensureNoTrack1Subscribers(parentCtx context.Context, } return err } - -func (aeh *AzureEventHubs) createContainerStorageClient() (*container.Client, error) { - options := container.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Telemetry: policy.TelemetryOptions{ - ApplicationID: "dapr-" + logger.DaprVersion, - }, - }, - } - - var ( - err error - client *container.Client - ) - // Use the global URL for Azure Storage - containerURL := fmt.Sprintf("https://%s.blob.%s/%s", aeh.metadata.StorageAccountName, "core.windows.net", aeh.metadata.StorageContainerName) - - if aeh.metadata.StorageConnectionString != "" { - // Authenticate with a connection string - client, err = container.NewClientFromConnectionString(aeh.metadata.StorageConnectionString, aeh.metadata.StorageContainerName, &options) - if err != nil { - return nil, fmt.Errorf("error creating Azure Storage client from connection string: %w", err) - } - } else if aeh.metadata.StorageAccountKey != "" { - // Authenticate with a shared key - credential, newSharedKeyErr := azblob.NewSharedKeyCredential(aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey) - if newSharedKeyErr != nil { - return nil, fmt.Errorf("invalid Azure Storage shared key credentials with error: %w", newSharedKeyErr) - } - client, err = container.NewClientWithSharedKeyCredential(containerURL, credential, &options) - if err != nil { - return nil, fmt.Errorf("error creating Azure Storage client from shared key credentials: %w", err) - } - } else { - // Use Azure AD - settings, err := azauth.NewEnvironmentSettings("storage", aeh.metadata.properties) - if err != nil { - return nil, err - } - credential, tokenErr := settings.GetTokenCredential() - if tokenErr != nil { - return nil, fmt.Errorf("invalid Azure Storage token credentials with error: %w", tokenErr) - } - client, err = container.NewClient(containerURL, credential, &options) - if err != nil { - return nil, fmt.Errorf("error creating Azure Storage client from token credentials: %w", err) - } - } - - return client, nil -} diff --git a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go index faa9e102b..b00a7df2f 100644 --- a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go +++ b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go @@ -27,8 +27,8 @@ import ( "go.uber.org/multierr" // Pub-Sub. - - pubsub_evethubs "github.com/dapr/components-contrib/pubsub/azure/eventhubs" + "github.com/dapr/components-contrib/pubsub" + pubsub_eventhubs "github.com/dapr/components-contrib/pubsub/azure/eventhubs" secretstore_env "github.com/dapr/components-contrib/secretstores/local/env" pubsub_loader "github.com/dapr/dapr/pkg/components/pubsub" secretstores_loader "github.com/dapr/dapr/pkg/components/secretstores" @@ -131,7 +131,7 @@ func TestEventhubs(t *testing.T) { client := sidecar.GetClient(ctx, sidecarName) // publish messages - ctx.Logf("Publishing messages. sidecarName: %s, topicName: %s", sidecarName, topicName) + logs := fmt.Sprintf("Published messages. sidecarName: %s, topicName: %s", sidecarName, topicName) var publishOptions dapr.PublishEventOption @@ -139,8 +139,8 @@ func TestEventhubs(t *testing.T) { publishOptions = dapr.PublishEventWithMetadata(metadata) } - for _, message := range messages { - ctx.Logf("Publishing: %q", message) + for i, message := range messages { + logs += fmt.Sprintf("\nMessage %d: %s", i, message) var err error if publishOptions != nil { @@ -149,8 +149,11 @@ func TestEventhubs(t *testing.T) { err = client.PublishEvent(ctx, pubsubName, topicName, message) } - require.NoError(ctx, err, "error publishing message") + require.NoErrorf(ctx, err, "error publishing message %s", message) } + + ctx.Log(logs) + return nil } } @@ -159,7 +162,7 @@ func TestEventhubs(t *testing.T) { return func(ctx flow.Context) error { // assert for messages for _, m := range messageWatchers { - m.Assert(ctx, 25*timeout) + m.Assert(ctx, 10*timeout) } return nil @@ -181,13 +184,19 @@ func TestEventhubs(t *testing.T) { messageWatchers.ExpectStrings(messages...) output, err := exec.Command("/bin/sh", "send-iot-device-events.sh", topicToBeCreated).Output() - assert.Nil(t, err, "Error in send-iot-device-events.sh.:\n%s", string(output)) + assert.NoErrorf(t, err, "Error in send-iot-device-events.sh.:\n%s", string(output)) return nil } } // Topic name for a IOT device is same as IOTHubName iotHubName := os.Getenv(iotHubNameEnvKey) + // Here so we can comment out tests as needed + _ = publishMessageAsDevice + _ = iotHubName + _ = consumerGroup4 + _ = consumerGroup5 + flow.New(t, "eventhubs certification"). // Test : single publisher, multiple subscriber with their own consumerID @@ -201,7 +210,8 @@ func TestEventhubs(t *testing.T) { embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), - componentRuntimeOptions(), + embedded.WithProfilePort(runtime.DefaultProfilePort), + componentRuntimeOptions(1), )). // Run subscriberApplication app2 @@ -215,8 +225,9 @@ func TestEventhubs(t *testing.T) { embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset), - componentRuntimeOptions(), + componentRuntimeOptions(2), )). + Step("wait", flow.Sleep(10*time.Second)). Step("publish messages to topic1", publishMessages(nil, sidecarName1, topicActiveName, consumerGroup1, consumerGroup2)). Step("publish messages to unUsedTopic", publishMessages(nil, sidecarName1, topicPassiveName)). Step("verify if app1 has recevied messages published to topic1", assertMessages(10*time.Second, consumerGroup1)). @@ -235,13 +246,15 @@ func TestEventhubs(t *testing.T) { embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*2), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*2), embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*2), - componentRuntimeOptions(), + componentRuntimeOptions(3), )). + Step("wait", flow.Sleep(10*time.Second)). // publish message in topic1 from two publisher apps, however there are two subscriber apps (app2,app3) with same consumerID - Step("publish messages to topic1", publishMessages(metadata, sidecarName1, topicActiveName, consumerGroup2)). - Step("publish messages to topic1", publishMessages(metadata1, sidecarName2, topicActiveName, consumerGroup2)). + Step("publish messages to topic1 from app1", publishMessages(metadata, sidecarName1, topicActiveName, consumerGroup2)). + Step("publish messages to topic1 from app2", publishMessages(metadata1, sidecarName2, topicActiveName, consumerGroup2)). Step("verify if app2, app3 together have recevied messages published to topic1", assertMessages(10*time.Second, consumerGroup2)). + // Test : Entitymanagement , Test partition key, in order processing with single publisher/subscriber // Run subscriberApplication app4 Step(app.Run(appID4, fmt.Sprintf(":%d", appPort+portOffset*3), @@ -254,7 +267,7 @@ func TestEventhubs(t *testing.T) { embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3), embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*3), - componentRuntimeOptions(), + componentRuntimeOptions(4), )). Step(fmt.Sprintf("publish messages to topicToBeCreated: %s", topicToBeCreated), publishMessages(metadata, sidecarName4, topicToBeCreated, consumerGroup4)). Step("verify if app4 has recevied messages published to newly created topic", assertMessages(10*time.Second, consumerGroup4)). @@ -270,22 +283,31 @@ func TestEventhubs(t *testing.T) { embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*4), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*4), embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*4), - componentRuntimeOptions(), + componentRuntimeOptions(5), )). Step("add expected IOT messages (simulate add message to iot)", publishMessageAsDevice(consumerGroup5)). Step("verify if app5 has recevied messages published to iot topic", assertMessages(40*time.Second, consumerGroup5)). Step("wait", flow.Sleep(5*time.Second)). + // cleanup azure assets created as part of tests Step("delete eventhub created as part of the eventhub management test", deleteEventhub). Run() } -func componentRuntimeOptions() []runtime.Option { +func componentRuntimeOptions(instance int) []runtime.Option { log := logger.NewLogger("dapr.components") + log.SetOutputLevel(logger.DebugLevel) pubsubRegistry := pubsub_loader.NewRegistry() pubsubRegistry.Logger = log - pubsubRegistry.RegisterComponent(pubsub_evethubs.NewAzureEventHubs, "azure.eventhubs") + pubsubRegistry.RegisterComponent(func(l logger.Logger) pubsub.PubSub { + l = l.WithFields(map[string]any{ + "component": "pubsub.azure.eventhubs", + "instance": instance, + }) + l.Infof("Instantiated log for instance %d", instance) + return pubsub_eventhubs.NewAzureEventHubs(l) + }, "azure.eventhubs") secretstoreRegistry := secretstores_loader.NewRegistry() secretstoreRegistry.Logger = log From 4aeb055ac04eefe613338e181ca783f5a33de1d0 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 23:18:16 +0000 Subject: [PATCH 26/42] More cert test fixes Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- tests/certification/pubsub/azure/eventhubs/eventhubs_test.go | 4 +++- .../pubsub/azure/eventhubs/send-iot-device-events.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go index b00a7df2f..6024c1ec0 100644 --- a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go +++ b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go @@ -162,7 +162,7 @@ func TestEventhubs(t *testing.T) { return func(ctx flow.Context) error { // assert for messages for _, m := range messageWatchers { - m.Assert(ctx, 10*timeout) + m.Assert(ctx, 15*timeout) } return nil @@ -269,6 +269,7 @@ func TestEventhubs(t *testing.T) { embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*3), componentRuntimeOptions(4), )). + Step("wait", flow.Sleep(10*time.Second)). Step(fmt.Sprintf("publish messages to topicToBeCreated: %s", topicToBeCreated), publishMessages(metadata, sidecarName4, topicToBeCreated, consumerGroup4)). Step("verify if app4 has recevied messages published to newly created topic", assertMessages(10*time.Second, consumerGroup4)). @@ -285,6 +286,7 @@ func TestEventhubs(t *testing.T) { embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*4), componentRuntimeOptions(5), )). + Step("wait", flow.Sleep(10*time.Second)). Step("add expected IOT messages (simulate add message to iot)", publishMessageAsDevice(consumerGroup5)). Step("verify if app5 has recevied messages published to iot topic", assertMessages(40*time.Second, consumerGroup5)). Step("wait", flow.Sleep(5*time.Second)). diff --git a/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh b/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh index 13662b2fc..8e93a34bc 100755 --- a/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh +++ b/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh @@ -30,7 +30,7 @@ fi az config set extension.use_dynamic_install=yes_without_prompt # login to azure -az login --service-principal -u $AzureCertificationServicePrincipalClientId -p $AzureCertificationServicePrincipalClientSecret --tenant $AzureCertificationTenantId +az login --service-principal -u $AzureCertificationServicePrincipalClientId -p $AzureCertificationServicePrincipalClientSecret --tenant $AzureCertificationTenantId > /dev/null # Create test device ID if not already present IOT_HUB_TEST_DEVICE_NAME="certification-test-device" From ab7f9aa509d682ac0364333e394382bed5c179ec Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 23:20:58 +0000 Subject: [PATCH 27/42] =?UTF-8?q?=F0=9F=92=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- pubsub/azure/eventhubs/eventhubs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 85c705ee3..7bfa452ee 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -157,7 +157,8 @@ func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPubl // Batch options batchOpts := &azeventhubs.EventDataBatchOptions{} if val := req.Metadata[contribMetadata.MaxBulkPubBytesKey]; val != "" { - maxBytes, err := strconv.ParseUint(val, 10, 63) + var maxBytes uint64 + maxBytes, err = strconv.ParseUint(val, 10, 63) if err == nil && maxBytes > 0 { batchOpts.MaxBytes = maxBytes } From d1fed3807ccb0922b4208d2f433b9288459232e6 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 20 Jan 2023 23:33:23 +0000 Subject: [PATCH 28/42] Updated Action to fix warning Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .github/workflows/certification.yml | 4 ++-- .github/workflows/conformance.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/certification.yml b/.github/workflows/certification.yml index 5afaae6cf..480cc73f6 100644 --- a/.github/workflows/certification.yml +++ b/.github/workflows/certification.yml @@ -134,7 +134,7 @@ jobs: - name: Create PR comment if: env.PR_NUMBER != '' - uses: artursouza/sticky-pull-request-comment@v2.2.0 + uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba with: header: ${{ github.run_id }} number: ${{ env.PR_NUMBER }} @@ -568,7 +568,7 @@ jobs: - name: Replace PR comment if: always() && env.PR_NUMBER != '' - uses: artursouza/sticky-pull-request-comment@v2.2.0 + uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba with: header: ${{ github.run_id }} number: ${{ env.PR_NUMBER }} diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 6d4f3ef7a..b48dc521c 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -162,7 +162,7 @@ jobs: - name: Create PR comment if: env.PR_NUMBER != '' - uses: artursouza/sticky-pull-request-comment@v2.2.0 + uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba with: header: ${{ github.run_id }} number: ${{ env.PR_NUMBER }} @@ -773,7 +773,7 @@ jobs: - name: Replace PR comment if: env.PR_NUMBER != '' - uses: artursouza/sticky-pull-request-comment@v2.2.0 + uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba with: header: ${{ github.run_id }} number: ${{ env.PR_NUMBER }} From d8e1afb8aeff8798c65d5926c7fe36d1823f64fb Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Sat, 21 Jan 2023 00:12:07 +0000 Subject: [PATCH 29/42] Tweaks to tests Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../pubsub/azure/eventhubs/eventhubs_test.go | 19 ++++++++++++------- .../azure/eventhubs/send-iot-device-events.sh | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go index 6024c1ec0..83a596394 100644 --- a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go +++ b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go @@ -171,7 +171,7 @@ func TestEventhubs(t *testing.T) { deleteEventhub := func(ctx flow.Context) error { output, err := exec.Command("/bin/sh", "delete-eventhub.sh", topicToBeCreated).Output() - assert.Nil(t, err, "Error in delete-eventhub.sh.:\n%s", string(output)) + assert.NoErrorf(t, err, "Error in delete-eventhub.sh.:\n%s", string(output)) return nil } @@ -192,10 +192,15 @@ func TestEventhubs(t *testing.T) { iotHubName := os.Getenv(iotHubNameEnvKey) // Here so we can comment out tests as needed - _ = publishMessageAsDevice - _ = iotHubName + _ = consumerGroup1 + _ = consumerGroup2 _ = consumerGroup4 _ = consumerGroup5 + _ = publishMessageAsDevice + _ = iotHubName + _ = metadata + _ = metadata1 + _ = publishMessages flow.New(t, "eventhubs certification"). @@ -286,12 +291,12 @@ func TestEventhubs(t *testing.T) { embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*4), componentRuntimeOptions(5), )). - Step("wait", flow.Sleep(10*time.Second)). - Step("add expected IOT messages (simulate add message to iot)", publishMessageAsDevice(consumerGroup5)). - Step("verify if app5 has recevied messages published to iot topic", assertMessages(40*time.Second, consumerGroup5)). - Step("wait", flow.Sleep(5*time.Second)). + Step("wait", flow.Sleep(20*time.Second)). + Step("add expected IoT messages (simulate add message to IoT Hub)", publishMessageAsDevice(consumerGroup5)). + Step("verify if app5 has recevied messages published to IoT topic", assertMessages(10*time.Second, consumerGroup5)). // cleanup azure assets created as part of tests + Step("wait", flow.Sleep(5*time.Second)). Step("delete eventhub created as part of the eventhub management test", deleteEventhub). Run() } diff --git a/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh b/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh index 8e93a34bc..13662b2fc 100755 --- a/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh +++ b/tests/certification/pubsub/azure/eventhubs/send-iot-device-events.sh @@ -30,7 +30,7 @@ fi az config set extension.use_dynamic_install=yes_without_prompt # login to azure -az login --service-principal -u $AzureCertificationServicePrincipalClientId -p $AzureCertificationServicePrincipalClientSecret --tenant $AzureCertificationTenantId > /dev/null +az login --service-principal -u $AzureCertificationServicePrincipalClientId -p $AzureCertificationServicePrincipalClientSecret --tenant $AzureCertificationTenantId # Create test device ID if not already present IOT_HUB_TEST_DEVICE_NAME="certification-test-device" From 857a865e4e21504fb4fd484da974fade99682718 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Sat, 21 Jan 2023 01:14:04 +0000 Subject: [PATCH 30/42] Moved implementation to shared package Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../azure/eventhubs/entity_management.go | 0 .../component/azure/eventhubs/eventhubs.go | 644 ++++++++++++++++++ .../azure/eventhubs/eventhubs_test.go | 41 +- .../component}/azure/eventhubs/events.go | 46 +- .../component}/azure/eventhubs/metadata.go | 7 +- .../azure/eventhubs/track1_upgrade.go | 0 pubsub/azure/eventhubs/eventhubs.go | 620 +---------------- .../eventhubs/eventhubs_integration_test.go | 16 +- 8 files changed, 702 insertions(+), 672 deletions(-) rename {pubsub => internal/component}/azure/eventhubs/entity_management.go (100%) create mode 100644 internal/component/azure/eventhubs/eventhubs.go rename {pubsub => internal/component}/azure/eventhubs/eventhubs_test.go (76%) rename {pubsub => internal/component}/azure/eventhubs/events.go (68%) rename {pubsub => internal/component}/azure/eventhubs/metadata.go (94%) rename {pubsub => internal/component}/azure/eventhubs/track1_upgrade.go (100%) diff --git a/pubsub/azure/eventhubs/entity_management.go b/internal/component/azure/eventhubs/entity_management.go similarity index 100% rename from pubsub/azure/eventhubs/entity_management.go rename to internal/component/azure/eventhubs/entity_management.go diff --git a/internal/component/azure/eventhubs/eventhubs.go b/internal/component/azure/eventhubs/eventhubs.go new file mode 100644 index 000000000..df7900117 --- /dev/null +++ b/internal/component/azure/eventhubs/eventhubs.go @@ -0,0 +1,644 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eventhubs + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "golang.org/x/exp/maps" + + azauth "github.com/dapr/components-contrib/internal/authentication/azure" + "github.com/dapr/kit/logger" + "github.com/dapr/kit/retry" +) + +// AzureEventHubs allows sending/receiving Azure Event Hubs events. +// This is an abstract class used by both the pubsub and binding components. +type AzureEventHubs struct { + metadata *azureEventHubsMetadata + logger logger.Logger + + backOffConfig retry.Config + producersLock *sync.RWMutex + producers map[string]*azeventhubs.ProducerClient + checkpointStoreCache azeventhubs.CheckpointStore + checkpointStoreLock *sync.RWMutex + + managementCreds azcore.TokenCredential + + // TODO(@ItalyPaleAle): Remove in Dapr 1.13 + isFailed *atomic.Bool +} + +// NewAzureEventHubs returns a new Azure Event hubs instance. +func NewAzureEventHubs(logger logger.Logger) *AzureEventHubs { + return &AzureEventHubs{ + logger: logger, + producersLock: &sync.RWMutex{}, + producers: make(map[string]*azeventhubs.ProducerClient, 1), + checkpointStoreLock: &sync.RWMutex{}, + isFailed: &atomic.Bool{}, + } +} + +// Init connects to Azure Event Hubs. +func (aeh *AzureEventHubs) Init(metadata map[string]string) error { + m, err := parseEventHubsMetadata(metadata, aeh.logger) + if err != nil { + return err + } + aeh.metadata = m + + if aeh.metadata.ConnectionString != "" { + // Connect using the connection string + hubName := hubNameFromConnString(aeh.metadata.ConnectionString) + if hubName != "" { + aeh.logger.Infof(`The provided connection string is specific to the Event Hub ("entity path") '%s'; publishing or subscribing to a topic that does not match this Event Hub will fail when attempted`, hubName) + } else { + aeh.logger.Infof(`The provided connection string does not contain an Event Hub name ("entity path"); the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored`) + } + + aeh.metadata.hubName = hubName + } else { + // Connect via Azure AD + var env azauth.EnvironmentSettings + env, err = azauth.NewEnvironmentSettings("eventhubs", metadata) + if err != nil { + return fmt.Errorf("failed to initialize Azure AD credentials: %w", err) + } + aeh.metadata.aadTokenProvider, err = env.GetTokenCredential() + if err != nil { + return fmt.Errorf("failed to get Azure AD token credentials provider: %w", err) + } + + aeh.logger.Info("connecting to Azure Event Hub using Azure AD; the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored") + + if aeh.metadata.EnableEntityManagement { + err = aeh.initEntityManagement() + if err != nil { + return fmt.Errorf("failed to initialize entity manager: %w", err) + } + } + } + + // Default retry configuration is used if no backOff properties are set + // backOff max retry config is set to 3, which means 3 retries by default + aeh.backOffConfig = retry.DefaultConfig() + aeh.backOffConfig.MaxRetries = 3 + err = retry.DecodeConfigWithPrefix(&aeh.backOffConfig, metadata, "backOff") + if err != nil { + return fmt.Errorf("failed to decode backoff configuration") + } + + return nil +} + +// Publish a batch of messages. +func (aeh *AzureEventHubs) Publish(ctx context.Context, topic string, messages []*azeventhubs.EventData, batchOpts *azeventhubs.EventDataBatchOptions) error { + // Get the producer client + client, err := aeh.getProducerClientForTopic(ctx, topic) + if err != nil { + return fmt.Errorf("error trying to establish a connection: %w", err) + } + + // Build the batch of messages + batch, err := client.NewEventDataBatch(ctx, batchOpts) + if err != nil { + return fmt.Errorf("error creating event batch: %w", err) + } + + // Add all messages + for _, msg := range messages { + err = batch.AddEventData(msg, nil) + if err != nil { + return fmt.Errorf("error adding messages to batch: %w", err) + } + } + + // Send the message + err = client.SendEventDataBatch(ctx, batch, nil) + if err != nil { + return fmt.Errorf("error publishing batch: %w", err) + } + + return nil +} + +// Subscribe receives data from Azure Event Hubs. +func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, topic string, getAllProperties bool, handler SubscribeHandler) (err error) { + if aeh.metadata.ConsumerGroup == "" { + return errors.New("property consumerID is required to subscribe to an Event Hub topic") + } + + // Get the processor client + processor, err := aeh.getProcessorForTopic(subscribeCtx, topic) + if err != nil { + return fmt.Errorf("error trying to establish a connection: %w", err) + } + + // Ensure that no subscriber using the old "track 1" SDK is active + // TODO(@ItalyPaleAle): Remove this for Dapr 1.13 + { + // If a previous topic already failed, no need to try with other topics, as we're about to panic anyways + if aeh.isFailed.Load() { + return errors.New("subscribing to another topic on this component failed and Dapr is scheduled to crash; will not try subscribing to a new topic") + } + + ctx, cancel := context.WithTimeout(subscribeCtx, 2*time.Minute) + err = aeh.ensureNoTrack1Subscribers(ctx, topic) + cancel() + if err != nil { + // If there's a timeout, it means that the other client was still active after the timeout + // In this case, we return an error here so Dapr can continue the initialization and report a "healthy" status (but this subscription won't be active) + // After 2 minutes, then, we panic, which ensures that during a rollout Kubernetes will see that this pod is unhealthy and re-creates that. Hopefully, by then other instances of the app will have been updated and no more locks will be present + if errors.Is(err, context.DeadlineExceeded) { + aeh.isFailed.Store(true) + errMsg := fmt.Sprintf("Another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr, and this is not supported. Please ensure that all applications subscribed to the same topic, with this consumer group, are using Dapr 1.10 or newer.", topic) + aeh.logger.Error(errMsg + " ⚠️⚠️⚠️ Dapr will crash in 2 minutes to force the orchestrator to restart the process after the rollout of other instances is complete.") + go func() { + time.Sleep(2 * time.Minute) + aeh.logger.Fatalf("Another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr, and this is not supported. Please ensure that all applications subscribed to the same topic, with this consumer group, are using Dapr 1.10 or newer.", topic) + }() + return fmt.Errorf("another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr", topic) + } + + // In case of other errors, just return the error + return fmt.Errorf("failed to check for subscribers using an old version of Dapr: %w", err) + } + } + + // This component has built-in retries because Event Hubs doesn't support N/ACK for messages + retryHandler := func(ctx context.Context, data []byte, metadata map[string]string) error { + b := aeh.backOffConfig.NewBackOffWithContext(subscribeCtx) + + mID := metadata[sysPropMessageID] + if mID == "" { + mID = "(nil)" + } + // This method is synchronous so no risk of race conditions if using side effects + var attempts int + retryerr := retry.NotifyRecover(func() error { + attempts++ + aeh.logger.Debugf("Processing EventHubs event %s/%s (attempt: %d)", topic, mID, attempts) + + if attempts > 1 { + metadata["dapr-attempt"] = strconv.Itoa(attempts) + } + + return handler(ctx, data, metadata) + }, b, func(_ error, _ time.Duration) { + aeh.logger.Warnf("Error processing EventHubs event: %s/%s. Retrying...", topic, mID) + }, func() { + aeh.logger.Warnf("Successfully processed EventHubs event after it previously failed: %s/%s", topic, mID) + }) + if retryerr != nil { + aeh.logger.Errorf("Too many failed attempts at processing Eventhubs event: %s/%s. Error: %v", topic, mID, err) + } + return retryerr + } + + // Get the subscribe handler + eventHandler := subscribeHandler(subscribeCtx, getAllProperties, retryHandler) + + // Process all partition clients as they come in + go func() { + for { + // This will block until a new partition client is available + // It returns nil if processor.Run terminates or if the context is canceled + partitionClient := processor.NextPartitionClient(subscribeCtx) + if partitionClient == nil { + return + } + aeh.logger.Debugf("Received client for partition %s", partitionClient.PartitionID()) + + // Once we get a partition client, process the events in a separate goroutine + go func() { + processErr := aeh.processEvents(subscribeCtx, topic, partitionClient, eventHandler) + // Do not log context.Canceled which happens at shutdown + if processErr != nil && !errors.Is(processErr, context.Canceled) { + aeh.logger.Errorf("Error processing events from partition client: %v", processErr) + } + }() + } + }() + + // Start the processor + go func() { + // This is a blocking call that runs until the context is canceled + err = processor.Run(subscribeCtx) + // Do not log context.Canceled which happens at shutdown + if err != nil && !errors.Is(err, context.Canceled) { + aeh.logger.Errorf("Error from event processor: %v", err) + } + }() + + return nil +} + +func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, topic string, partitionClient *azeventhubs.ProcessorPartitionClient, eventHandler func(e *azeventhubs.ReceivedEventData) error) error { + // At the end of the method we need to do some cleanup and close the partition client + defer func() { + closeCtx, closeCancel := context.WithTimeout(context.Background(), resourceGetTimeout) + defer closeCancel() + closeErr := partitionClient.Close(closeCtx) + if closeErr != nil { + aeh.logger.Errorf("Error while closing partition client: %v", closeErr) + } + }() + + // Loop to receive messages + var ( + ctx context.Context + cancel context.CancelFunc + events []*azeventhubs.ReceivedEventData + err error + ) + for { + // TODO: Support setting a batch size + const batchSize = 1 + ctx, cancel = context.WithTimeout(subscribeCtx, time.Minute) + events, err = partitionClient.ReceiveEvents(ctx, batchSize, nil) + cancel() + + // A DeadlineExceeded error means that the context timed out before we received the full batch of messages, and that's fine + if err != nil && !errors.Is(err, context.DeadlineExceeded) { + // If we get an error like ErrorCodeOwnershipLost, it means that the partition was rebalanced and we lost it + // We'll just stop this subscription and return + eventHubError := (*azeventhubs.Error)(nil) + if errors.As(err, &eventHubError) && eventHubError.Code == azeventhubs.ErrorCodeOwnershipLost { + aeh.logger.Debugf("Client lost ownership of partition %s for topic %s", partitionClient.PartitionID(), topic) + return nil + } + + return fmt.Errorf("error receiving events: %w", err) + } + + aeh.logger.Debugf("Received batch with %d events on topic %s, partition %s", len(events), topic, partitionClient.PartitionID()) + + if len(events) != 0 { + for _, event := range events { + // Process the event in its own goroutine + go eventHandler(event) + } + + // Update the checkpoint with the last event received. If we lose ownership of this partition or have to restart the next owner will start from this point. + // This context inherits from the background one in case subscriptionCtx gets canceled + ctx, cancel = context.WithTimeout(context.Background(), resourceCreationTimeout) + err = partitionClient.UpdateCheckpoint(ctx, events[len(events)-1]) + cancel() + if err != nil { + return fmt.Errorf("failed to update checkpoint: %w", err) + } + } + } +} + +func (aeh *AzureEventHubs) Close() (err error) { + // Acquire locks + aeh.checkpointStoreLock.Lock() + defer aeh.checkpointStoreLock.Unlock() + aeh.producersLock.Lock() + defer aeh.producersLock.Unlock() + + // Close all producers + wg := sync.WaitGroup{} + for _, producer := range aeh.producers { + if producer == nil { + continue + } + wg.Add(1) + go func(producer *azeventhubs.ProducerClient) { + closeCtx, closeCancel := context.WithTimeout(context.Background(), resourceGetTimeout) + defer closeCancel() + producer.Close(closeCtx) + wg.Done() + }(producer) + } + wg.Wait() + maps.Clear(aeh.producers) + + // Remove the cached checkpoint store and metadata + aeh.checkpointStoreCache = nil + aeh.metadata = nil + + return nil +} + +// Returns a producer client for a given topic. +// If the client doesn't exist in the cache, it will create one. +func (aeh *AzureEventHubs) getProducerClientForTopic(ctx context.Context, topic string) (client *azeventhubs.ProducerClient, err error) { + // Check if we have the producer client in the cache + aeh.producersLock.RLock() + client = aeh.producers[topic] + aeh.producersLock.RUnlock() + if client != nil { + return client, nil + } + + // After acquiring a write lock, check again if the producer exists in the cache just in case another goroutine created it in the meanwhile + aeh.producersLock.Lock() + defer aeh.producersLock.Unlock() + + client = aeh.producers[topic] + if client != nil { + return client, nil + } + + // Create a new entity if needed + if aeh.metadata.EnableEntityManagement { + err = aeh.ensureEventHubEntity(ctx, topic) + if err != nil { + return nil, fmt.Errorf("failed to create Event Hub entity %s: %w", topic, err) + } + } + + clientOpts := &azeventhubs.ProducerClientOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + } + + // Check if we're authenticating using a connection string + if aeh.metadata.ConnectionString != "" { + var connString string + connString, err = aeh.constructConnectionStringFromTopic(topic) + if err != nil { + return nil, err + } + client, err = azeventhubs.NewProducerClientFromConnectionString(connString, "", clientOpts) + if err != nil { + return nil, fmt.Errorf("unable to connect to Azure Event Hub using a connection string: %w", err) + } + } else { + // Use Azure AD + client, err = azeventhubs.NewProducerClient(aeh.metadata.EventHubNamespace, topic, aeh.metadata.aadTokenProvider, clientOpts) + if err != nil { + return nil, fmt.Errorf("unable to connect to Azure Event Hub using Azure AD: %w", err) + } + } + + // Store in the cache before returning it + aeh.producers[topic] = client + return client, nil +} + +// Creates a processor for a given topic. +func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic string) (*azeventhubs.Processor, error) { + // Get the checkpoint store + checkpointStore, err := aeh.getCheckpointStore(ctx) + if err != nil { + return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err) + } + + // Create a new entity if needed + if aeh.metadata.EnableEntityManagement { + // First ensure that the Event Hub entity exists + // We need to acquire a lock on producers, as creating a producer can perform the same operations + aeh.producersLock.Lock() + err = aeh.ensureEventHubEntity(ctx, topic) + aeh.producersLock.Unlock() + if err != nil { + return nil, fmt.Errorf("failed to create Event Hub entity %s: %w", topic, err) + } + + // Abuse on the lock on checkpoints which are used by all tasks creating processors + aeh.checkpointStoreLock.Lock() + err = aeh.ensureSubscription(ctx, topic) + aeh.checkpointStoreLock.Unlock() + if err != nil { + return nil, fmt.Errorf("failed to create Event Hub subscription to entity %s: %w", topic, err) + } + } + + // Create a consumer client + var consumerClient *azeventhubs.ConsumerClient + clientOpts := &azeventhubs.ConsumerClientOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + } + + // Check if we're authenticating using a connection string + if aeh.metadata.ConnectionString != "" { + var connString string + connString, err = aeh.constructConnectionStringFromTopic(topic) + if err != nil { + return nil, err + } + consumerClient, err = azeventhubs.NewConsumerClientFromConnectionString(connString, "", aeh.metadata.ConsumerGroup, clientOpts) + if err != nil { + return nil, fmt.Errorf("unable to connect to Azure Event Hub using a connection string: %w", err) + } + } else { + // Use Azure AD + consumerClient, err = azeventhubs.NewConsumerClient(aeh.metadata.EventHubNamespace, topic, aeh.metadata.ConsumerGroup, aeh.metadata.aadTokenProvider, clientOpts) + if err != nil { + return nil, fmt.Errorf("unable to connect to Azure Event Hub using Azure AD: %w", err) + } + } + + // Create the processor from the consumer client and checkpoint store + processor, err := azeventhubs.NewProcessor(consumerClient, checkpointStore, nil) + if err != nil { + return nil, fmt.Errorf("unable to create the processor: %w", err) + } + + return processor, nil +} + +// Returns the checkpoint store from the object. If it doesn't exist, it lazily initializes it. +func (aeh *AzureEventHubs) getCheckpointStore(ctx context.Context) (azeventhubs.CheckpointStore, error) { + // Check if we have the checkpoint store + aeh.checkpointStoreLock.RLock() + if aeh.checkpointStoreCache != nil { + aeh.checkpointStoreLock.RUnlock() + return aeh.checkpointStoreCache, nil + } + aeh.checkpointStoreLock.RUnlock() + + // After acquiring a write lock, check again if the checkpoint store exists in case another goroutine created it in the meanwhile + aeh.checkpointStoreLock.Lock() + defer aeh.checkpointStoreLock.Unlock() + + if aeh.checkpointStoreCache != nil { + return aeh.checkpointStoreCache, nil + } + + // Init the checkpoint store and store it in the object + var err error + aeh.checkpointStoreCache, err = aeh.createCheckpointStore(ctx) + if err != nil { + return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err) + } + + return aeh.checkpointStoreCache, nil +} + +// Initializes a new checkpoint store +func (aeh *AzureEventHubs) createCheckpointStore(ctx context.Context) (checkpointStore azeventhubs.CheckpointStore, err error) { + if aeh.metadata.StorageAccountName == "" { + return nil, errors.New("property storageAccountName is required to subscribe to an Event Hub topic") + } + if aeh.metadata.StorageContainerName == "" { + return nil, errors.New("property storageContainerName is required to subscribe to an Event Hub topic") + } + + // Ensure the container exists + err = aeh.ensureStorageContainer(ctx) + if err != nil { + return nil, err + } + + // Create the checkpoint store + checkpointStoreOpts := &checkpoints.BlobStoreOptions{ + ClientOptions: policy.ClientOptions{ + Telemetry: policy.TelemetryOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + }, + }, + } + if aeh.metadata.StorageConnectionString != "" { + // Authenticate with a connection string + checkpointStore, err = checkpoints.NewBlobStoreFromConnectionString(aeh.metadata.StorageConnectionString, aeh.metadata.StorageContainerName, checkpointStoreOpts) + if err != nil { + return nil, fmt.Errorf("error creating checkpointer from connection string: %w", err) + } + } else if aeh.metadata.StorageAccountKey != "" { + // Authenticate with a shared key + // TODO: This is a workaround in which we assemble a connection string until https://github.com/Azure/azure-sdk-for-go/issues/19842 is fixed + connString := fmt.Sprintf("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=core.windows.net", aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey) + checkpointStore, err = checkpoints.NewBlobStoreFromConnectionString(connString, aeh.metadata.StorageContainerName, checkpointStoreOpts) + if err != nil { + return nil, fmt.Errorf("error creating checkpointer from storage account credentials: %w", err) + } + } else { + // Use Azure AD + // If Event Hub is authenticated using a connection string, we can't use Azure AD here + if aeh.metadata.ConnectionString != "" { + return nil, errors.New("either one of storageConnectionString or storageAccountKey is required when subscribing to an Event Hub topic without using Azure AD") + } + // Use the global URL for Azure Storage + containerURL := fmt.Sprintf("https://%s.blob.%s/%s", aeh.metadata.StorageAccountName, "core.windows.net", aeh.metadata.StorageContainerName) + checkpointStore, err = checkpoints.NewBlobStore(containerURL, aeh.metadata.aadTokenProvider, checkpointStoreOpts) + if err != nil { + return nil, fmt.Errorf("error creating checkpointer from Azure AD credentials: %w", err) + } + } + + return checkpointStore, nil +} + +// Ensures that the container exists in the Azure Storage Account. +// This is done to preserve backwards-compatibility with Dapr 1.9, as the old checkpoint SDK created them automatically. +func (aeh *AzureEventHubs) ensureStorageContainer(parentCtx context.Context) error { + // Get a client to Azure Blob Storage + client, err := aeh.createStorageClient() + if err != nil { + return err + } + + // Create the container + // This will return an error if it already exists + ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) + defer cancel() + _, err = client.CreateContainer(ctx, aeh.metadata.StorageContainerName, &container.CreateOptions{ + // Default is private + Access: nil, + }) + if err != nil { + // Check if it's an Azure Storage error + resErr := &azcore.ResponseError{} + // If the container already exists, return no error + if errors.As(err, &resErr) && (resErr.ErrorCode == "ContainerAlreadyExists" || resErr.ErrorCode == "ResourceAlreadyExists") { + return nil + } + + return fmt.Errorf("failed to create Azure Storage container %s: %w", aeh.metadata.StorageContainerName, err) + } + + return nil +} + +// Creates a client to access Azure Blob Storage +func (aeh *AzureEventHubs) createStorageClient() (*azblob.Client, error) { + options := azblob.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Telemetry: policy.TelemetryOptions{ + ApplicationID: "dapr-" + logger.DaprVersion, + }, + }, + } + + var ( + err error + client *azblob.Client + ) + // Use the global URL for Azure Storage + accountURL := fmt.Sprintf("https://%s.blob.%s", aeh.metadata.StorageAccountName, "core.windows.net") + + if aeh.metadata.StorageConnectionString != "" { + // Authenticate with a connection string + client, err = azblob.NewClientFromConnectionString(aeh.metadata.StorageConnectionString, &options) + if err != nil { + return nil, fmt.Errorf("error creating Azure Storage client from connection string: %w", err) + } + } else if aeh.metadata.StorageAccountKey != "" { + // Authenticate with a shared key + credential, newSharedKeyErr := azblob.NewSharedKeyCredential(aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey) + if newSharedKeyErr != nil { + return nil, fmt.Errorf("invalid Azure Storage shared key credentials with error: %w", newSharedKeyErr) + } + client, err = azblob.NewClientWithSharedKeyCredential(accountURL, credential, &options) + if err != nil { + return nil, fmt.Errorf("error creating Azure Storage client from shared key credentials: %w", err) + } + } else { + // Use Azure AD + settings, err := azauth.NewEnvironmentSettings("storage", aeh.metadata.properties) + if err != nil { + return nil, err + } + credential, tokenErr := settings.GetTokenCredential() + if tokenErr != nil { + return nil, fmt.Errorf("invalid Azure Storage token credentials with error: %w", tokenErr) + } + client, err = azblob.NewClient(accountURL, credential, &options) + if err != nil { + return nil, fmt.Errorf("error creating Azure Storage client from token credentials: %w", err) + } + } + + return client, nil +} + +// Returns a connection string with the Event Hub name (entity path) set if not present. +func (aeh *AzureEventHubs) constructConnectionStringFromTopic(topic string) (string, error) { + if aeh.metadata.hubName != "" { + if aeh.metadata.hubName != topic { + return "", fmt.Errorf("the requested topic '%s' does not match the Event Hub name in the connection string", topic) + } + return aeh.metadata.ConnectionString, nil + } + + connString := aeh.metadata.ConnectionString + ";EntityPath=" + topic + return connString, nil +} diff --git a/pubsub/azure/eventhubs/eventhubs_test.go b/internal/component/azure/eventhubs/eventhubs_test.go similarity index 76% rename from pubsub/azure/eventhubs/eventhubs_test.go rename to internal/component/azure/eventhubs/eventhubs_test.go index 2cb58d924..b84167255 100644 --- a/pubsub/azure/eventhubs/eventhubs_test.go +++ b/internal/component/azure/eventhubs/eventhubs_test.go @@ -19,8 +19,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/dapr/components-contrib/metadata" - "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" ) @@ -28,9 +26,8 @@ var testLogger = logger.NewLogger("test") func TestParseEventHubsMetadata(t *testing.T) { t.Run("test valid connectionString configuration", func(t *testing.T) { - props := map[string]string{"connectionString": "fake"} + metadata := map[string]string{"connectionString": "fake"} - metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} m, err := parseEventHubsMetadata(metadata, testLogger) require.NoError(t, err) @@ -38,9 +35,8 @@ func TestParseEventHubsMetadata(t *testing.T) { }) t.Run("test namespace given", func(t *testing.T) { - props := map[string]string{"eventHubNamespace": "fake.servicebus.windows.net"} + metadata := map[string]string{"eventHubNamespace": "fake.servicebus.windows.net"} - metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} m, err := parseEventHubsMetadata(metadata, testLogger) require.NoError(t, err) @@ -48,9 +44,8 @@ func TestParseEventHubsMetadata(t *testing.T) { }) t.Run("test namespace adds FQDN", func(t *testing.T) { - props := map[string]string{"eventHubNamespace": "fake"} + metadata := map[string]string{"eventHubNamespace": "fake"} - metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} m, err := parseEventHubsMetadata(metadata, testLogger) require.NoError(t, err) @@ -58,12 +53,11 @@ func TestParseEventHubsMetadata(t *testing.T) { }) t.Run("test both connectionString and eventHubNamespace given", func(t *testing.T) { - props := map[string]string{ + metadata := map[string]string{ "connectionString": "fake", "eventHubNamespace": "fake", } - metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} _, err := parseEventHubsMetadata(metadata, testLogger) require.Error(t, err) @@ -71,9 +65,8 @@ func TestParseEventHubsMetadata(t *testing.T) { }) t.Run("test missing metadata", func(t *testing.T) { - props := map[string]string{} + metadata := map[string]string{} - metadata := pubsub.Metadata{Base: metadata.Base{Properties: props}} _, err := parseEventHubsMetadata(metadata, testLogger) require.Error(t, err) @@ -86,11 +79,9 @@ func TestConstructConnectionStringFromTopic(t *testing.T) { connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key" topic := "testHub" - metadata := pubsub.Metadata{Base: metadata.Base{ - Properties: map[string]string{ - "connectionString": connectionString, - }, - }} + metadata := map[string]string{ + "connectionString": connectionString, + } aeh := &AzureEventHubs{logger: testLogger} err := aeh.Init(metadata) require.NoError(t, err) @@ -104,11 +95,9 @@ func TestConstructConnectionStringFromTopic(t *testing.T) { connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key;EntityPath=testHub" topic := "testHub" - metadata := pubsub.Metadata{Base: metadata.Base{ - Properties: map[string]string{ - "connectionString": connectionString, - }, - }} + metadata := map[string]string{ + "connectionString": connectionString, + } aeh := &AzureEventHubs{logger: testLogger} err := aeh.Init(metadata) require.NoError(t, err) @@ -122,11 +111,9 @@ func TestConstructConnectionStringFromTopic(t *testing.T) { connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key;EntityPath=testHub" topic := "differentHub" - metadata := pubsub.Metadata{Base: metadata.Base{ - Properties: map[string]string{ - "connectionString": connectionString, - }, - }} + metadata := map[string]string{ + "connectionString": connectionString, + } aeh := &AzureEventHubs{logger: testLogger} err := aeh.Init(metadata) require.NoError(t, err) diff --git a/pubsub/azure/eventhubs/events.go b/internal/component/azure/eventhubs/events.go similarity index 68% rename from pubsub/azure/eventhubs/events.go rename to internal/component/azure/eventhubs/events.go index 248a6c4ee..7e189a776 100644 --- a/pubsub/azure/eventhubs/events.go +++ b/internal/component/azure/eventhubs/events.go @@ -18,16 +18,15 @@ package eventhubs import ( "context" "strconv" - "sync" "time" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/spf13/cast" - "golang.org/x/exp/maps" - - "github.com/dapr/components-contrib/pubsub" ) +// Type for the handler for messages coming in from the subscriptions. +type SubscribeHandler func(ctx context.Context, data []byte, metadata map[string]string) error + const ( // Event Hubs SystemProperties names for metadata passthrough. sysPropSequenceNumber = "x-opt-sequence-number" @@ -43,40 +42,23 @@ const ( sysPropMessageID = "message-id" ) -// Pool of map[string]string that we can use to optimize memory usage -var metadataPool = sync.Pool{ - New: func() any { - // Initial capacity is 5 which is the number of properties we normally find in a message - // The map can expand as needed, and when it's added back to the pool, we'll keep the larger size - m := make(map[string]string, 5) - return &m - }, -} - -func subscribeHandler(ctx context.Context, topic string, getAllProperties bool, handler pubsub.Handler) func(e *azeventhubs.ReceivedEventData) error { +func subscribeHandler(ctx context.Context, getAllProperties bool, handler SubscribeHandler) func(e *azeventhubs.ReceivedEventData) error { return func(e *azeventhubs.ReceivedEventData) error { - md := metadataPool.Get().(*map[string]string) - maps.Clear(*md) - defer metadataPool.Put(md) + // Allocate with an initial capacity of 10 which covers the common properties, also from IoT Hub + md := make(map[string]string, 10) - res := pubsub.NewMessage{ - Data: e.Body, - Topic: topic, - Metadata: *md, - } - - res.Metadata[sysPropSequenceNumber] = strconv.FormatInt(e.SequenceNumber, 10) + md[sysPropSequenceNumber] = strconv.FormatInt(e.SequenceNumber, 10) if e.EnqueuedTime != nil { - res.Metadata[sysPropEnqueuedTime] = e.EnqueuedTime.Format(time.RFC3339) + md[sysPropEnqueuedTime] = e.EnqueuedTime.Format(time.RFC3339) } if e.Offset != nil { - res.Metadata[sysPropOffset] = strconv.FormatInt(*e.Offset, 10) + md[sysPropOffset] = strconv.FormatInt(*e.Offset, 10) } if e.PartitionKey != nil { - res.Metadata[sysPropPartitionKey] = *e.PartitionKey + md[sysPropPartitionKey] = *e.PartitionKey } if e.MessageID != nil && *e.MessageID != "" { - res.Metadata[sysPropMessageID] = *e.MessageID + md[sysPropMessageID] = *e.MessageID } // Iterate through the system properties looking for those coming from IoT Hub @@ -88,7 +70,7 @@ func subscribeHandler(ctx context.Context, topic string, getAllProperties bool, sysPropIotHubConnectionAuthMethod, sysPropIotHubConnectionModuleID, sysPropIotHubEnqueuedTime: - addPropertyToMetadata(k, v, res.Metadata) + addPropertyToMetadata(k, v, md) default: // nop } @@ -97,11 +79,11 @@ func subscribeHandler(ctx context.Context, topic string, getAllProperties bool, // Added properties if any (includes application properties from Azure IoT Hub) if getAllProperties && len(e.Properties) > 0 { for k, v := range e.Properties { - addPropertyToMetadata(k, v, res.Metadata) + addPropertyToMetadata(k, v, md) } } - return handler(ctx, &res) + return handler(ctx, e.Body, md) } } diff --git a/pubsub/azure/eventhubs/metadata.go b/internal/component/azure/eventhubs/metadata.go similarity index 94% rename from pubsub/azure/eventhubs/metadata.go rename to internal/component/azure/eventhubs/metadata.go index 17fcd6566..7eb1ac5ab 100644 --- a/pubsub/azure/eventhubs/metadata.go +++ b/internal/component/azure/eventhubs/metadata.go @@ -22,7 +22,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/dapr/components-contrib/metadata" - "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" ) @@ -47,15 +46,15 @@ type azureEventHubsMetadata struct { properties map[string]string } -func parseEventHubsMetadata(meta pubsub.Metadata, log logger.Logger) (*azureEventHubsMetadata, error) { +func parseEventHubsMetadata(meta map[string]string, log logger.Logger) (*azureEventHubsMetadata, error) { var m azureEventHubsMetadata - err := metadata.DecodeMetadata(meta.Properties, &m) + err := metadata.DecodeMetadata(meta, &m) if err != nil { return nil, fmt.Errorf("failed to decode metada: %w", err) } // Store the raw properties in the object - m.properties = meta.Properties + m.properties = meta // One and only one of connectionString and eventHubNamespace is required if m.ConnectionString == "" && m.EventHubNamespace == "" { diff --git a/pubsub/azure/eventhubs/track1_upgrade.go b/internal/component/azure/eventhubs/track1_upgrade.go similarity index 100% rename from pubsub/azure/eventhubs/track1_upgrade.go rename to internal/component/azure/eventhubs/track1_upgrade.go diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 7bfa452ee..0596a662e 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -16,107 +16,33 @@ package eventhubs import ( "context" "errors" - "fmt" "strconv" - "sync" - "sync/atomic" - "time" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" - "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs/checkpoints" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" - "golang.org/x/exp/maps" - azauth "github.com/dapr/components-contrib/internal/authentication/azure" + impl "github.com/dapr/components-contrib/internal/component/azure/eventhubs" "github.com/dapr/components-contrib/internal/utils" contribMetadata "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" "github.com/dapr/kit/ptr" - "github.com/dapr/kit/retry" ) // AzureEventHubs allows sending/receiving Azure Event Hubs events. type AzureEventHubs struct { - metadata *azureEventHubsMetadata - logger logger.Logger - - backOffConfig retry.Config - producersLock *sync.RWMutex - producers map[string]*azeventhubs.ProducerClient - checkpointStoreCache azeventhubs.CheckpointStore - checkpointStoreLock *sync.RWMutex - - managementCreds azcore.TokenCredential - - // TODO(@ItalyPaleAle): Remove in Dapr 1.13 - isFailed *atomic.Bool + *impl.AzureEventHubs } // NewAzureEventHubs returns a new Azure Event hubs instance. func NewAzureEventHubs(logger logger.Logger) pubsub.PubSub { return &AzureEventHubs{ - logger: logger, - producersLock: &sync.RWMutex{}, - producers: make(map[string]*azeventhubs.ProducerClient, 1), - checkpointStoreLock: &sync.RWMutex{}, - isFailed: &atomic.Bool{}, + AzureEventHubs: impl.NewAzureEventHubs(logger), } } -// Init connects to Azure Event Hubs. +// Init the object. func (aeh *AzureEventHubs) Init(metadata pubsub.Metadata) error { - m, err := parseEventHubsMetadata(metadata, aeh.logger) - if err != nil { - return err - } - aeh.metadata = m - - if aeh.metadata.ConnectionString != "" { - // Connect using the connection string - hubName := hubNameFromConnString(aeh.metadata.ConnectionString) - if hubName != "" { - aeh.logger.Infof(`The provided connection string is specific to the Event Hub ("entity path") '%s'; publishing or subscribing to a topic that does not match this Event Hub will fail when attempted`, hubName) - } else { - aeh.logger.Infof(`The provided connection string does not contain an Event Hub name ("entity path"); the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored`) - } - - aeh.metadata.hubName = hubName - } else { - // Connect via Azure AD - var env azauth.EnvironmentSettings - env, err = azauth.NewEnvironmentSettings("eventhubs", metadata.Properties) - if err != nil { - return fmt.Errorf("failed to initialize Azure AD credentials: %w", err) - } - aeh.metadata.aadTokenProvider, err = env.GetTokenCredential() - if err != nil { - return fmt.Errorf("failed to get Azure AD token credentials provider: %w", err) - } - - aeh.logger.Info("connecting to Azure Event Hub using Azure AD; the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored") - - if aeh.metadata.EnableEntityManagement { - err = aeh.initEntityManagement() - if err != nil { - return fmt.Errorf("failed to initialize entity manager: %w", err) - } - } - } - - // Default retry configuration is used if no backOff properties are set - // backOff max retry config is set to 3, which means 3 retries by default - aeh.backOffConfig = retry.DefaultConfig() - aeh.backOffConfig.MaxRetries = 3 - err = retry.DecodeConfigWithPrefix(&aeh.backOffConfig, metadata.Properties, "backOff") - if err != nil { - return fmt.Errorf("failed to decode backoff configuration") - } - - return nil + return aeh.AzureEventHubs.Init(metadata.Properties) } func (aeh *AzureEventHubs) Features() []pubsub.Feature { @@ -142,7 +68,7 @@ func (aeh *AzureEventHubs) Publish(ctx context.Context, req *pubsub.PublishReque } // Publish the message - return aeh.doPublish(ctx, req.Topic, messages, batchOpts) + return aeh.AzureEventHubs.Publish(ctx, req.Topic, messages, batchOpts) } // BulkPublish sends data to Azure Event Hubs in bulk. @@ -183,7 +109,7 @@ func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPubl } // Publish the message - err = aeh.doPublish(ctx, req.Topic, messages, batchOpts) + err = aeh.AzureEventHubs.Publish(ctx, req.Topic, messages, batchOpts) if err != nil { // Partial success is not supported by Azure Event Hubs. // If an error occurs, all events are considered failed. @@ -193,535 +119,27 @@ func (aeh *AzureEventHubs) BulkPublish(ctx context.Context, req *pubsub.BulkPubl return pubsub.BulkPublishResponse{}, nil } -// Internal method used by Publish and BulkPublish to send messages -func (aeh *AzureEventHubs) doPublish(ctx context.Context, topic string, messages []*azeventhubs.EventData, batchOpts *azeventhubs.EventDataBatchOptions) error { - // Get the producer client - client, err := aeh.getProducerClientForTopic(ctx, topic) - if err != nil { - return fmt.Errorf("error trying to establish a connection: %w", err) - } - - // Build the batch of messages - batch, err := client.NewEventDataBatch(ctx, batchOpts) - if err != nil { - return fmt.Errorf("error creating event batch: %w", err) - } - - // Add all messages - for _, msg := range messages { - err = batch.AddEventData(msg, nil) - if err != nil { - return fmt.Errorf("error adding messages to batch: %w", err) - } - } - - // Send the message - err = client.SendEventDataBatch(ctx, batch, nil) - if err != nil { - return fmt.Errorf("error publishing batch: %w", err) - } - - return nil -} - // Subscribe receives data from Azure Event Hubs. -func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) (err error) { - if aeh.metadata.ConsumerGroup == "" { - return errors.New("property consumerID is required to subscribe to an Event Hub topic") - } - if req.Topic == "" { +func (aeh *AzureEventHubs) Subscribe(ctx context.Context, req pubsub.SubscribeRequest, handler pubsub.Handler) error { + topic := req.Topic + if topic == "" { return errors.New("parameter 'topic' is required") } // Check if requireAllProperties is set and is truthy getAllProperties := utils.IsTruthy(req.Metadata["requireAllProperties"]) - // Get the processor client - processor, err := aeh.getProcessorForTopic(subscribeCtx, req.Topic) - if err != nil { - return fmt.Errorf("error trying to establish a connection: %w", err) - } - - // Ensure that no subscriber using the old "track 1" SDK is active - // TODO(@ItalyPaleAle): Remove this for Dapr 1.13 - { - // If a previous topic already failed, no need to try with other topics, as we're about to panic anyways - if aeh.isFailed.Load() { - return errors.New("subscribing to another topic on this component failed and Dapr is scheduled to crash; will not try subscribing to a new topic") + // Start the subscription + return aeh.AzureEventHubs.Subscribe(ctx, topic, getAllProperties, func(ctx context.Context, data []byte, metadata map[string]string) error { + res := pubsub.NewMessage{ + Data: data, + Topic: topic, + Metadata: metadata, } - - ctx, cancel := context.WithTimeout(subscribeCtx, 2*time.Minute) - err = aeh.ensureNoTrack1Subscribers(ctx, req.Topic) - cancel() - if err != nil { - // If there's a timeout, it means that the other client was still active after the timeout - // In this case, we return an error here so Dapr can continue the initialization and report a "healthy" status (but this subscription won't be active) - // After 2 minutes, then, we panic, which ensures that during a rollout Kubernetes will see that this pod is unhealthy and re-creates that. Hopefully, by then other instances of the app will have been updated and no more locks will be present - if errors.Is(err, context.DeadlineExceeded) { - aeh.isFailed.Store(true) - errMsg := fmt.Sprintf("Another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr, and this is not supported. Please ensure that all applications subscribed to the same topic, with this consumer group, are using Dapr 1.10 or newer.", req.Topic) - aeh.logger.Error(errMsg + " ⚠️⚠️⚠️ Dapr will crash in 2 minutes to force the orchestrator to restart the process after the rollout of other instances is complete.") - go func() { - time.Sleep(2 * time.Minute) - aeh.logger.Fatalf("Another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr, and this is not supported. Please ensure that all applications subscribed to the same topic, with this consumer group, are using Dapr 1.10 or newer.", req.Topic) - }() - return fmt.Errorf("another instance is currently subscribed to the topic %s in this Event Hub using an old version of Dapr", req.Topic) - } - - // In case of other errors, just return the error - return fmt.Errorf("failed to check for subscribers using an old version of Dapr: %w", err) - } - } - - // This component has built-in retries because Event Hubs doesn't support N/ACK for messages - retryHandler := func(ctx context.Context, msg *pubsub.NewMessage) error { - b := aeh.backOffConfig.NewBackOffWithContext(subscribeCtx) - - mID := msg.Metadata[sysPropMessageID] - if mID == "" { - mID = "(nil)" - } - // This method is synchronous so no risk of race conditions if using side effects - var attempts int - retryerr := retry.NotifyRecover(func() error { - attempts++ - aeh.logger.Debugf("Processing EventHubs event %s/%s (attempt: %d)", msg.Topic, mID, attempts) - - if attempts > 1 { - msg.Metadata["dapr-attempt"] = strconv.Itoa(attempts) - } - - return handler(ctx, msg) - }, b, func(_ error, _ time.Duration) { - aeh.logger.Warnf("Error processing EventHubs event: %s/%s. Retrying...", msg.Topic, mID) - }, func() { - aeh.logger.Warnf("Successfully processed EventHubs event after it previously failed: %s/%s", msg.Topic, mID) - }) - if retryerr != nil { - aeh.logger.Errorf("Too many failed attempts at processing Eventhubs event: %s/%s. Error: %v", msg.Topic, mID, err) - } - return retryerr - } - - // Get the subscribe handler - eventHandler := subscribeHandler(subscribeCtx, req.Topic, getAllProperties, retryHandler) - - // Process all partition clients as they come in - go func() { - for { - // This will block until a new partition client is available - // It returns nil if processor.Run terminates or if the context is canceled - partitionClient := processor.NextPartitionClient(subscribeCtx) - if partitionClient == nil { - return - } - aeh.logger.Debugf("Received client for partition %s", partitionClient.PartitionID()) - - // Once we get a partition client, process the events in a separate goroutine - go func() { - processErr := aeh.processEvents(subscribeCtx, req.Topic, partitionClient, eventHandler) - // Do not log context.Canceled which happens at shutdown - if processErr != nil && !errors.Is(processErr, context.Canceled) { - aeh.logger.Errorf("Error processing events from partition client: %v", processErr) - } - }() - } - }() - - // Start the processor - go func() { - // This is a blocking call that runs until the context is canceled - err = processor.Run(subscribeCtx) - // Do not log context.Canceled which happens at shutdown - if err != nil && !errors.Is(err, context.Canceled) { - aeh.logger.Errorf("Error from event processor: %v", err) - } - }() - - return nil -} - -func (aeh *AzureEventHubs) processEvents(subscribeCtx context.Context, topic string, partitionClient *azeventhubs.ProcessorPartitionClient, eventHandler func(e *azeventhubs.ReceivedEventData) error) error { - // At the end of the method we need to do some cleanup and close the partition client - defer func() { - closeCtx, closeCancel := context.WithTimeout(context.Background(), resourceGetTimeout) - defer closeCancel() - closeErr := partitionClient.Close(closeCtx) - if closeErr != nil { - aeh.logger.Errorf("Error while closing partition client: %v", closeErr) - } - }() - - // Loop to receive messages - var ( - ctx context.Context - cancel context.CancelFunc - events []*azeventhubs.ReceivedEventData - err error - ) - for { - // TODO: Support setting a batch size - const batchSize = 1 - ctx, cancel = context.WithTimeout(subscribeCtx, time.Minute) - events, err = partitionClient.ReceiveEvents(ctx, batchSize, nil) - cancel() - - // A DeadlineExceeded error means that the context timed out before we received the full batch of messages, and that's fine - if err != nil && !errors.Is(err, context.DeadlineExceeded) { - // If we get an error like ErrorCodeOwnershipLost, it means that the partition was rebalanced and we lost it - // We'll just stop this subscription and return - eventHubError := (*azeventhubs.Error)(nil) - if errors.As(err, &eventHubError) && eventHubError.Code == azeventhubs.ErrorCodeOwnershipLost { - aeh.logger.Debugf("Client lost ownership of partition %s for topic %s", partitionClient.PartitionID(), topic) - return nil - } - - return fmt.Errorf("error receiving events: %w", err) - } - - aeh.logger.Debugf("Received batch with %d events on topic %s, partition %s", len(events), topic, partitionClient.PartitionID()) - - if len(events) != 0 { - for _, event := range events { - // Process the event in its own goroutine - go eventHandler(event) - } - - // Update the checkpoint with the last event received. If we lose ownership of this partition or have to restart the next owner will start from this point. - // This context inherits from the background one in case subscriptionCtx gets canceled - ctx, cancel = context.WithTimeout(context.Background(), resourceCreationTimeout) - err = partitionClient.UpdateCheckpoint(ctx, events[len(events)-1]) - cancel() - if err != nil { - return fmt.Errorf("failed to update checkpoint: %w", err) - } - } - } + return handler(ctx, &res) + }) } func (aeh *AzureEventHubs) Close() (err error) { - // Acquire locks - aeh.checkpointStoreLock.Lock() - defer aeh.checkpointStoreLock.Unlock() - aeh.producersLock.Lock() - defer aeh.producersLock.Unlock() - - // Close all producers - wg := sync.WaitGroup{} - for _, producer := range aeh.producers { - if producer == nil { - continue - } - wg.Add(1) - go func(producer *azeventhubs.ProducerClient) { - closeCtx, closeCancel := context.WithTimeout(context.Background(), resourceGetTimeout) - defer closeCancel() - producer.Close(closeCtx) - wg.Done() - }(producer) - } - wg.Wait() - maps.Clear(aeh.producers) - - // Remove the cached checkpoint store and metadata - aeh.checkpointStoreCache = nil - aeh.metadata = nil - - return nil -} - -// Returns a producer client for a given topic. -// If the client doesn't exist in the cache, it will create one. -func (aeh *AzureEventHubs) getProducerClientForTopic(ctx context.Context, topic string) (client *azeventhubs.ProducerClient, err error) { - // Check if we have the producer client in the cache - aeh.producersLock.RLock() - client = aeh.producers[topic] - aeh.producersLock.RUnlock() - if client != nil { - return client, nil - } - - // After acquiring a write lock, check again if the producer exists in the cache just in case another goroutine created it in the meanwhile - aeh.producersLock.Lock() - defer aeh.producersLock.Unlock() - - client = aeh.producers[topic] - if client != nil { - return client, nil - } - - // Create a new entity if needed - if aeh.metadata.EnableEntityManagement { - err = aeh.ensureEventHubEntity(ctx, topic) - if err != nil { - return nil, fmt.Errorf("failed to create Event Hub entity %s: %w", topic, err) - } - } - - clientOpts := &azeventhubs.ProducerClientOptions{ - ApplicationID: "dapr-" + logger.DaprVersion, - } - - // Check if we're authenticating using a connection string - if aeh.metadata.ConnectionString != "" { - var connString string - connString, err = aeh.constructConnectionStringFromTopic(topic) - if err != nil { - return nil, err - } - client, err = azeventhubs.NewProducerClientFromConnectionString(connString, "", clientOpts) - if err != nil { - return nil, fmt.Errorf("unable to connect to Azure Event Hub using a connection string: %w", err) - } - } else { - // Use Azure AD - client, err = azeventhubs.NewProducerClient(aeh.metadata.EventHubNamespace, topic, aeh.metadata.aadTokenProvider, clientOpts) - if err != nil { - return nil, fmt.Errorf("unable to connect to Azure Event Hub using Azure AD: %w", err) - } - } - - // Store in the cache before returning it - aeh.producers[topic] = client - return client, nil -} - -// Creates a processor for a given topic. -func (aeh *AzureEventHubs) getProcessorForTopic(ctx context.Context, topic string) (*azeventhubs.Processor, error) { - // Get the checkpoint store - checkpointStore, err := aeh.getCheckpointStore(ctx) - if err != nil { - return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err) - } - - // Create a new entity if needed - if aeh.metadata.EnableEntityManagement { - // First ensure that the Event Hub entity exists - // We need to acquire a lock on producers, as creating a producer can perform the same operations - aeh.producersLock.Lock() - err = aeh.ensureEventHubEntity(ctx, topic) - aeh.producersLock.Unlock() - if err != nil { - return nil, fmt.Errorf("failed to create Event Hub entity %s: %w", topic, err) - } - - // Abuse on the lock on checkpoints which are used by all tasks creating processors - aeh.checkpointStoreLock.Lock() - err = aeh.ensureSubscription(ctx, topic) - aeh.checkpointStoreLock.Unlock() - if err != nil { - return nil, fmt.Errorf("failed to create Event Hub subscription to entity %s: %w", topic, err) - } - } - - // Create a consumer client - var consumerClient *azeventhubs.ConsumerClient - clientOpts := &azeventhubs.ConsumerClientOptions{ - ApplicationID: "dapr-" + logger.DaprVersion, - } - - // Check if we're authenticating using a connection string - if aeh.metadata.ConnectionString != "" { - var connString string - connString, err = aeh.constructConnectionStringFromTopic(topic) - if err != nil { - return nil, err - } - consumerClient, err = azeventhubs.NewConsumerClientFromConnectionString(connString, "", aeh.metadata.ConsumerGroup, clientOpts) - if err != nil { - return nil, fmt.Errorf("unable to connect to Azure Event Hub using a connection string: %w", err) - } - } else { - // Use Azure AD - consumerClient, err = azeventhubs.NewConsumerClient(aeh.metadata.EventHubNamespace, topic, aeh.metadata.ConsumerGroup, aeh.metadata.aadTokenProvider, clientOpts) - if err != nil { - return nil, fmt.Errorf("unable to connect to Azure Event Hub using Azure AD: %w", err) - } - } - - // Create the processor from the consumer client and checkpoint store - processor, err := azeventhubs.NewProcessor(consumerClient, checkpointStore, nil) - if err != nil { - return nil, fmt.Errorf("unable to create the processor: %w", err) - } - - return processor, nil -} - -// Returns the checkpoint store from the object. If it doesn't exist, it lazily initializes it. -func (aeh *AzureEventHubs) getCheckpointStore(ctx context.Context) (azeventhubs.CheckpointStore, error) { - // Check if we have the checkpoint store - aeh.checkpointStoreLock.RLock() - if aeh.checkpointStoreCache != nil { - aeh.checkpointStoreLock.RUnlock() - return aeh.checkpointStoreCache, nil - } - aeh.checkpointStoreLock.RUnlock() - - // After acquiring a write lock, check again if the checkpoint store exists in case another goroutine created it in the meanwhile - aeh.checkpointStoreLock.Lock() - defer aeh.checkpointStoreLock.Unlock() - - if aeh.checkpointStoreCache != nil { - return aeh.checkpointStoreCache, nil - } - - // Init the checkpoint store and store it in the object - var err error - aeh.checkpointStoreCache, err = aeh.createCheckpointStore(ctx) - if err != nil { - return nil, fmt.Errorf("unable to connect to the checkpoint store: %w", err) - } - - return aeh.checkpointStoreCache, nil -} - -// Initializes a new checkpoint store -func (aeh *AzureEventHubs) createCheckpointStore(ctx context.Context) (checkpointStore azeventhubs.CheckpointStore, err error) { - if aeh.metadata.StorageAccountName == "" { - return nil, errors.New("property storageAccountName is required to subscribe to an Event Hub topic") - } - if aeh.metadata.StorageContainerName == "" { - return nil, errors.New("property storageContainerName is required to subscribe to an Event Hub topic") - } - - // Ensure the container exists - err = aeh.ensureStorageContainer(ctx) - if err != nil { - return nil, err - } - - // Create the checkpoint store - checkpointStoreOpts := &checkpoints.BlobStoreOptions{ - ClientOptions: policy.ClientOptions{ - Telemetry: policy.TelemetryOptions{ - ApplicationID: "dapr-" + logger.DaprVersion, - }, - }, - } - if aeh.metadata.StorageConnectionString != "" { - // Authenticate with a connection string - checkpointStore, err = checkpoints.NewBlobStoreFromConnectionString(aeh.metadata.StorageConnectionString, aeh.metadata.StorageContainerName, checkpointStoreOpts) - if err != nil { - return nil, fmt.Errorf("error creating checkpointer from connection string: %w", err) - } - } else if aeh.metadata.StorageAccountKey != "" { - // Authenticate with a shared key - // TODO: This is a workaround in which we assemble a connection string until https://github.com/Azure/azure-sdk-for-go/issues/19842 is fixed - connString := fmt.Sprintf("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=core.windows.net", aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey) - checkpointStore, err = checkpoints.NewBlobStoreFromConnectionString(connString, aeh.metadata.StorageContainerName, checkpointStoreOpts) - if err != nil { - return nil, fmt.Errorf("error creating checkpointer from storage account credentials: %w", err) - } - } else { - // Use Azure AD - // If Event Hub is authenticated using a connection string, we can't use Azure AD here - if aeh.metadata.ConnectionString != "" { - return nil, errors.New("either one of storageConnectionString or storageAccountKey is required when subscribing to an Event Hub topic without using Azure AD") - } - // Use the global URL for Azure Storage - containerURL := fmt.Sprintf("https://%s.blob.%s/%s", aeh.metadata.StorageAccountName, "core.windows.net", aeh.metadata.StorageContainerName) - checkpointStore, err = checkpoints.NewBlobStore(containerURL, aeh.metadata.aadTokenProvider, checkpointStoreOpts) - if err != nil { - return nil, fmt.Errorf("error creating checkpointer from Azure AD credentials: %w", err) - } - } - - return checkpointStore, nil -} - -// Ensures that the container exists in the Azure Storage Account. -// This is done to preserve backwards-compatibility with Dapr 1.9, as the old checkpoint SDK created them automatically. -func (aeh *AzureEventHubs) ensureStorageContainer(parentCtx context.Context) error { - // Get a client to Azure Blob Storage - client, err := aeh.createStorageClient() - if err != nil { - return err - } - - // Create the container - // This will return an error if it already exists - ctx, cancel := context.WithTimeout(parentCtx, resourceCreationTimeout) - defer cancel() - _, err = client.CreateContainer(ctx, aeh.metadata.StorageContainerName, &container.CreateOptions{ - // Default is private - Access: nil, - }) - if err != nil { - // Check if it's an Azure Storage error - resErr := &azcore.ResponseError{} - // If the container already exists, return no error - if errors.As(err, &resErr) && (resErr.ErrorCode == "ContainerAlreadyExists" || resErr.ErrorCode == "ResourceAlreadyExists") { - return nil - } - - return fmt.Errorf("failed to create Azure Storage container %s: %w", aeh.metadata.StorageContainerName, err) - } - - return nil -} - -// Creates a client to access Azure Blob Storage -func (aeh *AzureEventHubs) createStorageClient() (*azblob.Client, error) { - options := azblob.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Telemetry: policy.TelemetryOptions{ - ApplicationID: "dapr-" + logger.DaprVersion, - }, - }, - } - - var ( - err error - client *azblob.Client - ) - // Use the global URL for Azure Storage - accountURL := fmt.Sprintf("https://%s.blob.%s", aeh.metadata.StorageAccountName, "core.windows.net") - - if aeh.metadata.StorageConnectionString != "" { - // Authenticate with a connection string - client, err = azblob.NewClientFromConnectionString(aeh.metadata.StorageConnectionString, &options) - if err != nil { - return nil, fmt.Errorf("error creating Azure Storage client from connection string: %w", err) - } - } else if aeh.metadata.StorageAccountKey != "" { - // Authenticate with a shared key - credential, newSharedKeyErr := azblob.NewSharedKeyCredential(aeh.metadata.StorageAccountName, aeh.metadata.StorageAccountKey) - if newSharedKeyErr != nil { - return nil, fmt.Errorf("invalid Azure Storage shared key credentials with error: %w", newSharedKeyErr) - } - client, err = azblob.NewClientWithSharedKeyCredential(accountURL, credential, &options) - if err != nil { - return nil, fmt.Errorf("error creating Azure Storage client from shared key credentials: %w", err) - } - } else { - // Use Azure AD - settings, err := azauth.NewEnvironmentSettings("storage", aeh.metadata.properties) - if err != nil { - return nil, err - } - credential, tokenErr := settings.GetTokenCredential() - if tokenErr != nil { - return nil, fmt.Errorf("invalid Azure Storage token credentials with error: %w", tokenErr) - } - client, err = azblob.NewClient(accountURL, credential, &options) - if err != nil { - return nil, fmt.Errorf("error creating Azure Storage client from token credentials: %w", err) - } - } - - return client, nil -} - -// Returns a connection string with the Event Hub name (entity path) set if not present. -func (aeh *AzureEventHubs) constructConnectionStringFromTopic(topic string) (string, error) { - if aeh.metadata.hubName != "" { - if aeh.metadata.hubName != topic { - return "", fmt.Errorf("the requested topic '%s' does not match the Event Hub name in the connection string", topic) - } - return aeh.metadata.ConnectionString, nil - } - - connString := aeh.metadata.ConnectionString + ";EntityPath=" + topic - return connString, nil + return aeh.AzureEventHubs.Close() } diff --git a/pubsub/azure/eventhubs/eventhubs_integration_test.go b/pubsub/azure/eventhubs/eventhubs_integration_test.go index 0df138a27..a3d7ccca2 100644 --- a/pubsub/azure/eventhubs/eventhubs_integration_test.go +++ b/pubsub/azure/eventhubs/eventhubs_integration_test.go @@ -110,14 +110,14 @@ func testReadIotHubEvents(t *testing.T) { // Verify expected IoT Hub device event metadata exists // TODO: add device messages than can populate the sysPropPartitionKey and sysPropIotHubConnectionModuleID metadata - assert.Contains(t, r.Metadata, sysPropSequenceNumber, "IoT device event missing: %s", sysPropSequenceNumber) - assert.Contains(t, r.Metadata, sysPropEnqueuedTime, "IoT device event missing: %s", sysPropEnqueuedTime) - assert.Contains(t, r.Metadata, sysPropOffset, "IoT device event missing: %s", sysPropOffset) - assert.Contains(t, r.Metadata, sysPropIotHubDeviceConnectionID, "IoT device event missing: %s", sysPropIotHubDeviceConnectionID) - assert.Contains(t, r.Metadata, sysPropIotHubAuthGenerationID, "IoT device event missing: %s", sysPropIotHubAuthGenerationID) - assert.Contains(t, r.Metadata, sysPropIotHubConnectionAuthMethod, "IoT device event missing: %s", sysPropIotHubConnectionAuthMethod) - assert.Contains(t, r.Metadata, sysPropIotHubEnqueuedTime, "IoT device event missing: %s", sysPropIotHubEnqueuedTime) - assert.Contains(t, r.Metadata, sysPropMessageID, "IoT device event missing: %s", sysPropMessageID) + assert.Contains(t, r.Metadata, "x-opt-sequence-number", "IoT device event missing: %s", "x-opt-sequence-number") + assert.Contains(t, r.Metadata, "x-opt-enqueued-time", "IoT device event missing: %s", "x-opt-enqueued-time") + assert.Contains(t, r.Metadata, "x-opt-offset", "IoT device event missing: %s", "x-opt-offset") + assert.Contains(t, r.Metadata, "iothub-connection-device-id", "IoT device event missing: %s", "iothub-connection-device-id") + assert.Contains(t, r.Metadata, "iothub-connection-auth-generation-id", "IoT device event missing: %s", "iothub-connection-auth-generation-id") + assert.Contains(t, r.Metadata, "iothub-connection-auth-method", "IoT device event missing: %s", "iothub-connection-auth-method") + assert.Contains(t, r.Metadata, "iothub-enqueuedtime", "IoT device event missing: %s", "iothub-enqueuedtime") + assert.Contains(t, r.Metadata, "message-id", "IoT device event missing: %s", "message-id") // Verify sent custom application property is received in IoT Hub device event metadata assert.Contains(t, r.Metadata, applicationProperty, "IoT device event missing: %s", applicationProperty) From 232c8d813fa88821b47f08e73bc03eca9f5d4117 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Sat, 21 Jan 2023 19:28:00 +0000 Subject: [PATCH 31/42] Working on binding Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- bindings/azure/eventhubs/eventhubs.go | 428 ++--------------- bindings/azure/eventhubs/eventhubs.go.old | 446 ++++++++++++++++++ .../eventhubs/eventhubs_integration_test.go | 64 +-- bindings/azure/eventhubs/eventhubs_test.go | 122 ----- .../component/azure/eventhubs/eventhubs.go | 46 +- .../azure/eventhubs/eventhubs_test.go | 10 +- .../component/azure/eventhubs/metadata.go | 39 +- pubsub/azure/eventhubs/eventhubs.go | 3 +- .../components/binding/iothub/eventhubs.yaml | 2 - .../binding/serviceprincipal/eventhubs.yaml | 2 - tests/conformance/bindings/bindings.go | 2 +- 11 files changed, 587 insertions(+), 577 deletions(-) create mode 100644 bindings/azure/eventhubs/eventhubs.go.old delete mode 100644 bindings/azure/eventhubs/eventhubs_test.go diff --git a/bindings/azure/eventhubs/eventhubs.go b/bindings/azure/eventhubs/eventhubs.go index 52eac099e..0be77490c 100644 --- a/bindings/azure/eventhubs/eventhubs.go +++ b/bindings/azure/eventhubs/eventhubs.go @@ -15,432 +15,80 @@ package eventhubs import ( "context" - "errors" - "fmt" - "strconv" - "strings" - "time" - "github.com/Azure/azure-amqp-common-go/v4/aad" - "github.com/Azure/azure-amqp-common-go/v4/conn" - eventhub "github.com/Azure/azure-event-hubs-go/v3" - "github.com/Azure/azure-event-hubs-go/v3/eph" - "github.com/Azure/azure-event-hubs-go/v3/storage" - "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/dapr/components-contrib/bindings" - azauth "github.com/dapr/components-contrib/internal/authentication/azure" + impl "github.com/dapr/components-contrib/internal/component/azure/eventhubs" "github.com/dapr/kit/logger" + "github.com/dapr/kit/ptr" ) -const ( - // metadata. - connectionString = "connectionString" - - // required by subscriber. - consumerGroup = "consumerGroup" - storageAccountName = "storageAccountName" - storageAccountKey = "storageAccountKey" - storageContainerName = "storageContainerName" - - // optional. - partitionKeyName = "partitionKey" - partitionIDName = "partitionID" - hubName = "eventHub" - hubNamespaceName = "eventHubNamespace" - - // errors. - hubConnectionInitErrorMsg = "error: creating eventHub hub client" - invalidConnectionStringErrorMsg = "error: connectionString is invalid" - missingHubNamespaceErrorMsg = "error: connectionString or eventHubNamespace is required" - missingHubNameErrorMsg = "error: connectionString or eventHub is required" - missingStorageAccountNameErrorMsg = "error: storageAccountName is a required attribute" - missingStorageAccountKeyErrorMsg = "error: storageAccountKey is a required attribute when connectionString is provided" - missingStorageContainerNameErrorMsg = "error: storageContainerName is a required attribute" - missingConsumerGroupErrorMsg = "error: consumerGroup is a required attribute" - - // Event Hubs SystemProperties names for metadata passthrough. - sysPropSequenceNumber = "x-opt-sequence-number" - sysPropEnqueuedTime = "x-opt-enqueued-time" - sysPropOffset = "x-opt-offset" - sysPropPartitionID = "x-opt-partition-id" - sysPropPartitionKey = "x-opt-partition-key" - sysPropIotHubDeviceConnectionID = "iothub-connection-device-id" - sysPropIotHubAuthGenerationID = "iothub-connection-auth-generation-id" - sysPropIotHubConnectionAuthMethod = "iothub-connection-auth-method" - sysPropIotHubConnectionModuleID = "iothub-connection-module-id" - sysPropIotHubEnqueuedTime = "iothub-enqueuedtime" - sysPropMessageID = "message-id" -) - -func readHandler(ctx context.Context, e *eventhub.Event, handler bindings.Handler) error { - res := bindings.ReadResponse{Data: e.Data, Metadata: map[string]string{}} - if e.SystemProperties.SequenceNumber != nil { - res.Metadata[sysPropSequenceNumber] = strconv.FormatInt(*e.SystemProperties.SequenceNumber, 10) - } - if e.SystemProperties.EnqueuedTime != nil { - res.Metadata[sysPropEnqueuedTime] = e.SystemProperties.EnqueuedTime.Format(time.RFC3339) - } - if e.SystemProperties.Offset != nil { - res.Metadata[sysPropOffset] = strconv.FormatInt(*e.SystemProperties.Offset, 10) - } - // According to azure-event-hubs-go docs, this will always be nil. - if e.SystemProperties.PartitionID != nil { - res.Metadata[sysPropPartitionID] = strconv.Itoa(int(*e.SystemProperties.PartitionID)) - } - // The following metadata properties are only present if event was generated by Azure IoT Hub. - if e.SystemProperties.PartitionKey != nil { - res.Metadata[sysPropPartitionKey] = *e.SystemProperties.PartitionKey - } - if e.SystemProperties.IoTHubDeviceConnectionID != nil { - res.Metadata[sysPropIotHubDeviceConnectionID] = *e.SystemProperties.IoTHubDeviceConnectionID - } - if e.SystemProperties.IoTHubAuthGenerationID != nil { - res.Metadata[sysPropIotHubAuthGenerationID] = *e.SystemProperties.IoTHubAuthGenerationID - } - if e.SystemProperties.IoTHubConnectionAuthMethod != nil { - res.Metadata[sysPropIotHubConnectionAuthMethod] = *e.SystemProperties.IoTHubConnectionAuthMethod - } - if e.SystemProperties.IoTHubConnectionModuleID != nil { - res.Metadata[sysPropIotHubConnectionModuleID] = *e.SystemProperties.IoTHubConnectionModuleID - } - if e.SystemProperties.IoTHubEnqueuedTime != nil { - res.Metadata[sysPropIotHubEnqueuedTime] = e.SystemProperties.IoTHubEnqueuedTime.Format(time.RFC3339) - } - // azure-event-hubs-go SDK pulls out the AMQP message-id property to the Event.ID property, map it from there. - if e.ID != "" { - res.Metadata[sysPropMessageID] = e.ID - } - _, err := handler(ctx, &res) - - return err -} - // AzureEventHubs allows sending/receiving Azure Event Hubs events. type AzureEventHubs struct { - hub *eventhub.Hub - metadata *azureEventHubsMetadata - eventHubSettings azauth.EnvironmentSettings - tokenProvider *aad.TokenProvider - storageCredential azblob.Credential - azureEnvironment *azure.Environment - logger logger.Logger - userAgent string -} - -type azureEventHubsMetadata struct { - connectionString string - consumerGroup string - storageAccountName string - storageAccountKey string - storageContainerName string - partitionID string - partitionKey string - eventHubName string - eventHubNamespaceName string -} - -func (m azureEventHubsMetadata) partitioned() bool { - return m.partitionID != "" + *impl.AzureEventHubs } // NewAzureEventHubs returns a new Azure Event hubs instance. func NewAzureEventHubs(logger logger.Logger) bindings.InputOutputBinding { - return &AzureEventHubs{logger: logger} -} - -func validate(connectionString string) error { - _, err := conn.ParsedConnectionFromStr(connectionString) - return err -} - -func (a *AzureEventHubs) getStoragePrefixString() (string, error) { - hubName, err := a.validateAndGetHubName() - if err != nil { - return "", err + return &AzureEventHubs{ + AzureEventHubs: impl.NewAzureEventHubs(logger, true), } - - // empty string in the end of slice to have a suffix "-". - return strings.Join([]string{"dapr", hubName, a.metadata.consumerGroup, ""}, "-"), nil -} - -func (a *AzureEventHubs) validateAndGetHubName() (string, error) { - hubName := a.metadata.eventHubName - if hubName == "" { - parsed, err := conn.ParsedConnectionFromStr(a.metadata.connectionString) - if err != nil { - return "", err - } - hubName = parsed.HubName - } - return hubName, nil } // Init performs metadata init. func (a *AzureEventHubs) Init(metadata bindings.Metadata) error { - m, err := parseMetadata(metadata) - if err != nil { - return err - } - - a.userAgent = "dapr-" + logger.DaprVersion - - a.metadata = m - if a.metadata.connectionString != "" { - // Validate connectionString. - validateErr := validate(a.metadata.connectionString) - if validateErr != nil { - return errors.New(invalidConnectionStringErrorMsg) - } - - var hub *eventhub.Hub - // Create partitioned sender if the partitionID is configured. - if a.metadata.partitioned() { - hub, err = eventhub.NewHubFromConnectionString(a.metadata.connectionString, - eventhub.HubWithPartitionedSender(a.metadata.partitionID), - eventhub.HubWithUserAgent(a.userAgent), - ) - } else { - hub, err = eventhub.NewHubFromConnectionString(a.metadata.connectionString, - eventhub.HubWithUserAgent(a.userAgent), - ) - } - - if err != nil { - return fmt.Errorf("unable to connect to azure event hubs: %w", err) - } - a.hub = hub - } else { - // Connect via AAD. - settings, sErr := azauth.NewEnvironmentSettings(azauth.AzureEventHubsResourceName, metadata.Properties) - if sErr != nil { - return sErr - } - a.eventHubSettings = settings - tokenProvider, tokenErr := a.eventHubSettings.GetAMQPTokenProvider() - if tokenErr != nil { - return fmt.Errorf("%s %w", hubConnectionInitErrorMsg, err) - } - a.tokenProvider = tokenProvider - - a.hub, err = eventhub.NewHub(a.metadata.eventHubNamespaceName, a.metadata.eventHubName, a.tokenProvider, eventhub.HubWithUserAgent(a.userAgent)) - if err != nil { - return fmt.Errorf("unable to connect to azure event hubs: %w", err) - } - } - - // connect to the storage account. - if m.storageAccountKey != "" { - metadata.Properties["accountKey"] = m.storageAccountKey - } - a.storageCredential, a.azureEnvironment, err = azauth.GetAzureStorageBlobCredentials(a.logger, m.storageAccountName, metadata.Properties) - if err != nil { - return fmt.Errorf("invalid credentials with error: %w", err) - } - - return nil -} - -func parseMetadata(meta bindings.Metadata) (*azureEventHubsMetadata, error) { - m := &azureEventHubsMetadata{} - - if val, ok := meta.Properties[connectionString]; ok && val != "" { - m.connectionString = val - } - - if val, ok := meta.Properties[storageAccountName]; ok && val != "" { - m.storageAccountName = val - } else { - return m, errors.New(missingStorageAccountNameErrorMsg) - } - - if val, ok := meta.Properties[storageAccountKey]; ok && val != "" { - m.storageAccountKey = val - } else if m.connectionString != "" { - return m, errors.New(missingStorageAccountKeyErrorMsg) - } - - if val, ok := meta.Properties[storageContainerName]; ok && val != "" { - m.storageContainerName = val - } else { - return m, errors.New(missingStorageContainerNameErrorMsg) - } - - if val, ok := meta.Properties[consumerGroup]; ok && val != "" { - m.consumerGroup = val - } else { - return m, errors.New(missingConsumerGroupErrorMsg) - } - - if val, ok := meta.Properties[partitionKeyName]; ok { - m.partitionKey = val - } - - if val, ok := meta.Properties[partitionIDName]; ok { - m.partitionID = val - } - - if val, ok := meta.Properties[hubName]; ok { - m.eventHubName = val - } else if m.connectionString == "" { - return m, errors.New(missingHubNameErrorMsg) - } - - if val, ok := meta.Properties[hubNamespaceName]; ok { - m.eventHubNamespaceName = val - } else if m.connectionString == "" { - return m, errors.New(missingHubNamespaceErrorMsg) - } - - return m, nil + return a.AzureEventHubs.Init(metadata.Properties) } func (a *AzureEventHubs) Operations() []bindings.OperationKind { - return []bindings.OperationKind{bindings.CreateOperation} + return []bindings.OperationKind{ + bindings.CreateOperation, + } } // Write posts an event hubs message. func (a *AzureEventHubs) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*bindings.InvokeResponse, error) { - event := &eventhub.Event{ - Data: req.Data, + // Get the partition key and content type + batchOpts := &azeventhubs.EventDataBatchOptions{} + if pk := req.Metadata["partitionKey"]; pk != "" { + batchOpts.PartitionKey = &pk + } + var contentType *string + if ct := req.Metadata["contentType"]; ct != "" { + contentType = ptr.Of(req.Metadata["contentType"]) } - // Send partitionKey in event. - if a.metadata.partitionKey != "" { - event.PartitionKey = &a.metadata.partitionKey - } else { - partitionKey, ok := req.Metadata[partitionKeyName] - if partitionKey != "" && ok { - event.PartitionKey = &partitionKey - } + // Publish the message + messages := []*azeventhubs.EventData{ + { + Body: req.Data, + ContentType: contentType, + }, } - err := a.hub.Send(ctx, event) + // Publish the message + err := a.AzureEventHubs.Publish(ctx, a.AzureEventHubs.EventHubName(), messages, batchOpts) if err != nil { return nil, err } - return nil, nil } -// Read gets messages from eventhubs in a non-blocking way. +// Read gets messages from Event Hubs in a non-blocking way. func (a *AzureEventHubs) Read(ctx context.Context, handler bindings.Handler) error { - if !a.metadata.partitioned() { - if err := a.RegisterEventProcessor(ctx, handler); err != nil { - return err + // Start the subscription + // This is non-blocking + return a.AzureEventHubs.Subscribe(ctx, a.AzureEventHubs.EventHubName(), false, func(ctx context.Context, data []byte, metadata map[string]string) error { + res := bindings.ReadResponse{ + Data: data, + Metadata: metadata, } - } else { - if err := a.RegisterPartitionedEventProcessor(ctx, handler); err != nil { - return err - } - } - - go func() { - // Wait for context to be canceled then close the connection - <-ctx.Done() - a.Close() - }() - - return nil + _, hErr := handler(ctx, &res) + return hErr + }) } -// RegisterPartitionedEventProcessor - receive eventhub messages by partitionID. -func (a *AzureEventHubs) RegisterPartitionedEventProcessor(ctx context.Context, handler bindings.Handler) error { - runtimeInfo, err := a.hub.GetRuntimeInformation(ctx) - if err != nil { - return err - } - - callback := func(c context.Context, event *eventhub.Event) error { - if event == nil { - return nil - } - return readHandler(c, event, handler) - } - - ops := []eventhub.ReceiveOption{ - eventhub.ReceiveWithLatestOffset(), - } - - if a.metadata.consumerGroup != "" { - a.logger.Infof("eventhubs: using consumer group %s", a.metadata.consumerGroup) - ops = append(ops, eventhub.ReceiveWithConsumerGroup(a.metadata.consumerGroup)) - } - - if contains(runtimeInfo.PartitionIDs, a.metadata.partitionID) { - a.logger.Infof("eventhubs: using partition id %s", a.metadata.partitionID) - - _, err := a.hub.Receive(ctx, a.metadata.partitionID, callback, ops...) - if err != nil { - return err - } - } - - return nil -} - -// RegisterEventProcessor - receive eventhub messages by eventprocessor -// host by balancing partitions. -func (a *AzureEventHubs) RegisterEventProcessor(ctx context.Context, handler bindings.Handler) error { - storagePrefix, err := a.getStoragePrefixString() - if err != nil { - return err - } - - leaserPrefixOpt := storage.WithPrefixInBlobPath(storagePrefix) - leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(a.storageCredential, a.metadata.storageAccountName, a.metadata.storageContainerName, *a.azureEnvironment, leaserPrefixOpt) - if err != nil { - return err - } - - var processor *eph.EventProcessorHost - if a.metadata.connectionString != "" { - processor, err = eph.NewFromConnectionString(ctx, a.metadata.connectionString, leaserCheckpointer, leaserCheckpointer, eph.WithNoBanner(), eph.WithConsumerGroup(a.metadata.consumerGroup)) - if err != nil { - return err - } - } else { - // AAD connection. - processor, err = eph.New(ctx, a.metadata.eventHubNamespaceName, a.metadata.eventHubName, a.tokenProvider, leaserCheckpointer, leaserCheckpointer, eph.WithNoBanner(), eph.WithConsumerGroup(a.metadata.consumerGroup)) - if err != nil { - return err - } - a.logger.Debugf("processor initialized via AAD for eventHubName %s", a.metadata.eventHubName) - } - - _, err = processor.RegisterHandler( - ctx, - func(c context.Context, event *eventhub.Event) error { - return readHandler(c, event, handler) - }, - ) - if err != nil { - return err - } - - err = processor.StartNonBlocking(ctx) - if err != nil { - return err - } - - return nil -} - -func contains(arr []string, str string) bool { - for _, a := range arr { - if a == str { - return true - } - } - - return false -} - -func (a *AzureEventHubs) Close() (err error) { - // Use a background context because the connection context may be canceled already - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - err = a.hub.Close(ctx) - cancel() - return err +func (a *AzureEventHubs) Close() error { + return a.AzureEventHubs.Close() } diff --git a/bindings/azure/eventhubs/eventhubs.go.old b/bindings/azure/eventhubs/eventhubs.go.old new file mode 100644 index 000000000..52eac099e --- /dev/null +++ b/bindings/azure/eventhubs/eventhubs.go.old @@ -0,0 +1,446 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eventhubs + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/Azure/azure-amqp-common-go/v4/aad" + "github.com/Azure/azure-amqp-common-go/v4/conn" + eventhub "github.com/Azure/azure-event-hubs-go/v3" + "github.com/Azure/azure-event-hubs-go/v3/eph" + "github.com/Azure/azure-event-hubs-go/v3/storage" + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/go-autorest/autorest/azure" + + "github.com/dapr/components-contrib/bindings" + azauth "github.com/dapr/components-contrib/internal/authentication/azure" + "github.com/dapr/kit/logger" +) + +const ( + // metadata. + connectionString = "connectionString" + + // required by subscriber. + consumerGroup = "consumerGroup" + storageAccountName = "storageAccountName" + storageAccountKey = "storageAccountKey" + storageContainerName = "storageContainerName" + + // optional. + partitionKeyName = "partitionKey" + partitionIDName = "partitionID" + hubName = "eventHub" + hubNamespaceName = "eventHubNamespace" + + // errors. + hubConnectionInitErrorMsg = "error: creating eventHub hub client" + invalidConnectionStringErrorMsg = "error: connectionString is invalid" + missingHubNamespaceErrorMsg = "error: connectionString or eventHubNamespace is required" + missingHubNameErrorMsg = "error: connectionString or eventHub is required" + missingStorageAccountNameErrorMsg = "error: storageAccountName is a required attribute" + missingStorageAccountKeyErrorMsg = "error: storageAccountKey is a required attribute when connectionString is provided" + missingStorageContainerNameErrorMsg = "error: storageContainerName is a required attribute" + missingConsumerGroupErrorMsg = "error: consumerGroup is a required attribute" + + // Event Hubs SystemProperties names for metadata passthrough. + sysPropSequenceNumber = "x-opt-sequence-number" + sysPropEnqueuedTime = "x-opt-enqueued-time" + sysPropOffset = "x-opt-offset" + sysPropPartitionID = "x-opt-partition-id" + sysPropPartitionKey = "x-opt-partition-key" + sysPropIotHubDeviceConnectionID = "iothub-connection-device-id" + sysPropIotHubAuthGenerationID = "iothub-connection-auth-generation-id" + sysPropIotHubConnectionAuthMethod = "iothub-connection-auth-method" + sysPropIotHubConnectionModuleID = "iothub-connection-module-id" + sysPropIotHubEnqueuedTime = "iothub-enqueuedtime" + sysPropMessageID = "message-id" +) + +func readHandler(ctx context.Context, e *eventhub.Event, handler bindings.Handler) error { + res := bindings.ReadResponse{Data: e.Data, Metadata: map[string]string{}} + if e.SystemProperties.SequenceNumber != nil { + res.Metadata[sysPropSequenceNumber] = strconv.FormatInt(*e.SystemProperties.SequenceNumber, 10) + } + if e.SystemProperties.EnqueuedTime != nil { + res.Metadata[sysPropEnqueuedTime] = e.SystemProperties.EnqueuedTime.Format(time.RFC3339) + } + if e.SystemProperties.Offset != nil { + res.Metadata[sysPropOffset] = strconv.FormatInt(*e.SystemProperties.Offset, 10) + } + // According to azure-event-hubs-go docs, this will always be nil. + if e.SystemProperties.PartitionID != nil { + res.Metadata[sysPropPartitionID] = strconv.Itoa(int(*e.SystemProperties.PartitionID)) + } + // The following metadata properties are only present if event was generated by Azure IoT Hub. + if e.SystemProperties.PartitionKey != nil { + res.Metadata[sysPropPartitionKey] = *e.SystemProperties.PartitionKey + } + if e.SystemProperties.IoTHubDeviceConnectionID != nil { + res.Metadata[sysPropIotHubDeviceConnectionID] = *e.SystemProperties.IoTHubDeviceConnectionID + } + if e.SystemProperties.IoTHubAuthGenerationID != nil { + res.Metadata[sysPropIotHubAuthGenerationID] = *e.SystemProperties.IoTHubAuthGenerationID + } + if e.SystemProperties.IoTHubConnectionAuthMethod != nil { + res.Metadata[sysPropIotHubConnectionAuthMethod] = *e.SystemProperties.IoTHubConnectionAuthMethod + } + if e.SystemProperties.IoTHubConnectionModuleID != nil { + res.Metadata[sysPropIotHubConnectionModuleID] = *e.SystemProperties.IoTHubConnectionModuleID + } + if e.SystemProperties.IoTHubEnqueuedTime != nil { + res.Metadata[sysPropIotHubEnqueuedTime] = e.SystemProperties.IoTHubEnqueuedTime.Format(time.RFC3339) + } + // azure-event-hubs-go SDK pulls out the AMQP message-id property to the Event.ID property, map it from there. + if e.ID != "" { + res.Metadata[sysPropMessageID] = e.ID + } + _, err := handler(ctx, &res) + + return err +} + +// AzureEventHubs allows sending/receiving Azure Event Hubs events. +type AzureEventHubs struct { + hub *eventhub.Hub + metadata *azureEventHubsMetadata + eventHubSettings azauth.EnvironmentSettings + tokenProvider *aad.TokenProvider + storageCredential azblob.Credential + azureEnvironment *azure.Environment + logger logger.Logger + userAgent string +} + +type azureEventHubsMetadata struct { + connectionString string + consumerGroup string + storageAccountName string + storageAccountKey string + storageContainerName string + partitionID string + partitionKey string + eventHubName string + eventHubNamespaceName string +} + +func (m azureEventHubsMetadata) partitioned() bool { + return m.partitionID != "" +} + +// NewAzureEventHubs returns a new Azure Event hubs instance. +func NewAzureEventHubs(logger logger.Logger) bindings.InputOutputBinding { + return &AzureEventHubs{logger: logger} +} + +func validate(connectionString string) error { + _, err := conn.ParsedConnectionFromStr(connectionString) + return err +} + +func (a *AzureEventHubs) getStoragePrefixString() (string, error) { + hubName, err := a.validateAndGetHubName() + if err != nil { + return "", err + } + + // empty string in the end of slice to have a suffix "-". + return strings.Join([]string{"dapr", hubName, a.metadata.consumerGroup, ""}, "-"), nil +} + +func (a *AzureEventHubs) validateAndGetHubName() (string, error) { + hubName := a.metadata.eventHubName + if hubName == "" { + parsed, err := conn.ParsedConnectionFromStr(a.metadata.connectionString) + if err != nil { + return "", err + } + hubName = parsed.HubName + } + return hubName, nil +} + +// Init performs metadata init. +func (a *AzureEventHubs) Init(metadata bindings.Metadata) error { + m, err := parseMetadata(metadata) + if err != nil { + return err + } + + a.userAgent = "dapr-" + logger.DaprVersion + + a.metadata = m + if a.metadata.connectionString != "" { + // Validate connectionString. + validateErr := validate(a.metadata.connectionString) + if validateErr != nil { + return errors.New(invalidConnectionStringErrorMsg) + } + + var hub *eventhub.Hub + // Create partitioned sender if the partitionID is configured. + if a.metadata.partitioned() { + hub, err = eventhub.NewHubFromConnectionString(a.metadata.connectionString, + eventhub.HubWithPartitionedSender(a.metadata.partitionID), + eventhub.HubWithUserAgent(a.userAgent), + ) + } else { + hub, err = eventhub.NewHubFromConnectionString(a.metadata.connectionString, + eventhub.HubWithUserAgent(a.userAgent), + ) + } + + if err != nil { + return fmt.Errorf("unable to connect to azure event hubs: %w", err) + } + a.hub = hub + } else { + // Connect via AAD. + settings, sErr := azauth.NewEnvironmentSettings(azauth.AzureEventHubsResourceName, metadata.Properties) + if sErr != nil { + return sErr + } + a.eventHubSettings = settings + tokenProvider, tokenErr := a.eventHubSettings.GetAMQPTokenProvider() + if tokenErr != nil { + return fmt.Errorf("%s %w", hubConnectionInitErrorMsg, err) + } + a.tokenProvider = tokenProvider + + a.hub, err = eventhub.NewHub(a.metadata.eventHubNamespaceName, a.metadata.eventHubName, a.tokenProvider, eventhub.HubWithUserAgent(a.userAgent)) + if err != nil { + return fmt.Errorf("unable to connect to azure event hubs: %w", err) + } + } + + // connect to the storage account. + if m.storageAccountKey != "" { + metadata.Properties["accountKey"] = m.storageAccountKey + } + a.storageCredential, a.azureEnvironment, err = azauth.GetAzureStorageBlobCredentials(a.logger, m.storageAccountName, metadata.Properties) + if err != nil { + return fmt.Errorf("invalid credentials with error: %w", err) + } + + return nil +} + +func parseMetadata(meta bindings.Metadata) (*azureEventHubsMetadata, error) { + m := &azureEventHubsMetadata{} + + if val, ok := meta.Properties[connectionString]; ok && val != "" { + m.connectionString = val + } + + if val, ok := meta.Properties[storageAccountName]; ok && val != "" { + m.storageAccountName = val + } else { + return m, errors.New(missingStorageAccountNameErrorMsg) + } + + if val, ok := meta.Properties[storageAccountKey]; ok && val != "" { + m.storageAccountKey = val + } else if m.connectionString != "" { + return m, errors.New(missingStorageAccountKeyErrorMsg) + } + + if val, ok := meta.Properties[storageContainerName]; ok && val != "" { + m.storageContainerName = val + } else { + return m, errors.New(missingStorageContainerNameErrorMsg) + } + + if val, ok := meta.Properties[consumerGroup]; ok && val != "" { + m.consumerGroup = val + } else { + return m, errors.New(missingConsumerGroupErrorMsg) + } + + if val, ok := meta.Properties[partitionKeyName]; ok { + m.partitionKey = val + } + + if val, ok := meta.Properties[partitionIDName]; ok { + m.partitionID = val + } + + if val, ok := meta.Properties[hubName]; ok { + m.eventHubName = val + } else if m.connectionString == "" { + return m, errors.New(missingHubNameErrorMsg) + } + + if val, ok := meta.Properties[hubNamespaceName]; ok { + m.eventHubNamespaceName = val + } else if m.connectionString == "" { + return m, errors.New(missingHubNamespaceErrorMsg) + } + + return m, nil +} + +func (a *AzureEventHubs) Operations() []bindings.OperationKind { + return []bindings.OperationKind{bindings.CreateOperation} +} + +// Write posts an event hubs message. +func (a *AzureEventHubs) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*bindings.InvokeResponse, error) { + event := &eventhub.Event{ + Data: req.Data, + } + + // Send partitionKey in event. + if a.metadata.partitionKey != "" { + event.PartitionKey = &a.metadata.partitionKey + } else { + partitionKey, ok := req.Metadata[partitionKeyName] + if partitionKey != "" && ok { + event.PartitionKey = &partitionKey + } + } + + err := a.hub.Send(ctx, event) + if err != nil { + return nil, err + } + + return nil, nil +} + +// Read gets messages from eventhubs in a non-blocking way. +func (a *AzureEventHubs) Read(ctx context.Context, handler bindings.Handler) error { + if !a.metadata.partitioned() { + if err := a.RegisterEventProcessor(ctx, handler); err != nil { + return err + } + } else { + if err := a.RegisterPartitionedEventProcessor(ctx, handler); err != nil { + return err + } + } + + go func() { + // Wait for context to be canceled then close the connection + <-ctx.Done() + a.Close() + }() + + return nil +} + +// RegisterPartitionedEventProcessor - receive eventhub messages by partitionID. +func (a *AzureEventHubs) RegisterPartitionedEventProcessor(ctx context.Context, handler bindings.Handler) error { + runtimeInfo, err := a.hub.GetRuntimeInformation(ctx) + if err != nil { + return err + } + + callback := func(c context.Context, event *eventhub.Event) error { + if event == nil { + return nil + } + return readHandler(c, event, handler) + } + + ops := []eventhub.ReceiveOption{ + eventhub.ReceiveWithLatestOffset(), + } + + if a.metadata.consumerGroup != "" { + a.logger.Infof("eventhubs: using consumer group %s", a.metadata.consumerGroup) + ops = append(ops, eventhub.ReceiveWithConsumerGroup(a.metadata.consumerGroup)) + } + + if contains(runtimeInfo.PartitionIDs, a.metadata.partitionID) { + a.logger.Infof("eventhubs: using partition id %s", a.metadata.partitionID) + + _, err := a.hub.Receive(ctx, a.metadata.partitionID, callback, ops...) + if err != nil { + return err + } + } + + return nil +} + +// RegisterEventProcessor - receive eventhub messages by eventprocessor +// host by balancing partitions. +func (a *AzureEventHubs) RegisterEventProcessor(ctx context.Context, handler bindings.Handler) error { + storagePrefix, err := a.getStoragePrefixString() + if err != nil { + return err + } + + leaserPrefixOpt := storage.WithPrefixInBlobPath(storagePrefix) + leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(a.storageCredential, a.metadata.storageAccountName, a.metadata.storageContainerName, *a.azureEnvironment, leaserPrefixOpt) + if err != nil { + return err + } + + var processor *eph.EventProcessorHost + if a.metadata.connectionString != "" { + processor, err = eph.NewFromConnectionString(ctx, a.metadata.connectionString, leaserCheckpointer, leaserCheckpointer, eph.WithNoBanner(), eph.WithConsumerGroup(a.metadata.consumerGroup)) + if err != nil { + return err + } + } else { + // AAD connection. + processor, err = eph.New(ctx, a.metadata.eventHubNamespaceName, a.metadata.eventHubName, a.tokenProvider, leaserCheckpointer, leaserCheckpointer, eph.WithNoBanner(), eph.WithConsumerGroup(a.metadata.consumerGroup)) + if err != nil { + return err + } + a.logger.Debugf("processor initialized via AAD for eventHubName %s", a.metadata.eventHubName) + } + + _, err = processor.RegisterHandler( + ctx, + func(c context.Context, event *eventhub.Event) error { + return readHandler(c, event, handler) + }, + ) + if err != nil { + return err + } + + err = processor.StartNonBlocking(ctx) + if err != nil { + return err + } + + return nil +} + +func contains(arr []string, str string) bool { + for _, a := range arr { + if a == str { + return true + } + } + + return false +} + +func (a *AzureEventHubs) Close() (err error) { + // Use a background context because the connection context may be canceled already + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err = a.hub.Close(ctx) + cancel() + return err +} diff --git a/bindings/azure/eventhubs/eventhubs_integration_test.go b/bindings/azure/eventhubs/eventhubs_integration_test.go index 1af516143..2eede303c 100644 --- a/bindings/azure/eventhubs/eventhubs_integration_test.go +++ b/bindings/azure/eventhubs/eventhubs_integration_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/dapr/components-contrib/bindings" "github.com/dapr/components-contrib/metadata" @@ -61,11 +62,11 @@ func createIotHubBindingsMetadata() bindings.Metadata { metadata := bindings.Metadata{ Base: metadata.Base{ Properties: map[string]string{ - connectionString: os.Getenv(iotHubConnectionStringEnvKey), - consumerGroup: os.Getenv(iotHubConsumerGroupEnvKey), - storageAccountName: os.Getenv(storageAccountNameEnvKey), - storageAccountKey: os.Getenv(storageAccountKeyEnvKey), - storageContainerName: testStorageContainerName, + "connectionString": os.Getenv(iotHubConnectionStringEnvKey), + "consumerGroup": os.Getenv(iotHubConsumerGroupEnvKey), + "storageAccountName": os.Getenv(storageAccountNameEnvKey), + "storageAccountKey": os.Getenv(storageAccountKeyEnvKey), + "storageContainerName": testStorageContainerName, }, }, } @@ -77,14 +78,14 @@ func createEventHubsBindingsAADMetadata() bindings.Metadata { metadata := bindings.Metadata{ Base: metadata.Base{ Properties: map[string]string{ - consumerGroup: os.Getenv(eventHubsBindingsConsumerGroupEnvKey), - storageAccountName: os.Getenv(azureBlobStorageAccountEnvKey), - storageContainerName: os.Getenv(eventHubsBindingsContainerEnvKey), - "eventHub": os.Getenv(eventHubBindingsHubEnvKey), - "eventHubNamespace": os.Getenv(eventHubBindingsNamespaceEnvKey), - "azureTenantId": os.Getenv(azureTenantIdEnvKey), - "azureClientId": os.Getenv(azureServicePrincipalClientIdEnvKey), - "azureClientSecret": os.Getenv(azureServicePrincipalClientSecretEnvKey), + "consumerGroup": os.Getenv(eventHubsBindingsConsumerGroupEnvKey), + "storageAccountName": os.Getenv(azureBlobStorageAccountEnvKey), + "storageContainerName": os.Getenv(eventHubsBindingsContainerEnvKey), + "eventHub": os.Getenv(eventHubBindingsHubEnvKey), + "eventHubNamespace": os.Getenv(eventHubBindingsNamespaceEnvKey), + "azureTenantId": os.Getenv(azureTenantIdEnvKey), + "azureClientId": os.Getenv(azureServicePrincipalClientIdEnvKey), + "azureClientSecret": os.Getenv(azureServicePrincipalClientSecretEnvKey), }, }, } @@ -96,18 +97,19 @@ func testEventHubsBindingsAADAuthentication(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - logger := logger.NewLogger("bindings.azure.eventhubs.integration.test") + log := logger.NewLogger("bindings.azure.eventhubs.integration.test") + log.SetOutputLevel(logger.DebugLevel) metadata := createEventHubsBindingsAADMetadata() - eventHubsBindings := NewAzureEventHubs(logger) + eventHubsBindings := NewAzureEventHubs(log) err := eventHubsBindings.Init(metadata) - assert.NoError(t, err) + require.NoError(t, err) req := &bindings.InvokeRequest{ Data: []byte("Integration test message"), } _, err = eventHubsBindings.Invoke(ctx, req) - assert.NoError(t, err) + require.NoError(t, err) // Setup Read binding to capture readResponses in a closure so that test asserts can be // performed on the main thread, including the case where the handler is never invoked. @@ -118,13 +120,13 @@ func testEventHubsBindingsAADAuthentication(t *testing.T) { } _, err = eventHubsBindings.Invoke(ctx, req) - assert.NoError(t, err) + require.NoError(t, err) eventHubsBindings.Read(ctx, handler) time.Sleep(1 * time.Second) _, err = eventHubsBindings.Invoke(ctx, req) - assert.NoError(t, err) + require.NoError(t, err) // Note: azure-event-hubs-go SDK defaultLeasePersistenceInterval is 5s // Sleep long enough so that the azure event hubs SDK has time to persist updated checkpoints @@ -132,9 +134,9 @@ func testEventHubsBindingsAADAuthentication(t *testing.T) { time.Sleep(10 * time.Second) assert.Greater(t, len(readResponses), 0, "Failed to receive any EventHub events") - logger.Infof("Received %d messages", len(readResponses)) + log.Infof("Received %d messages", len(readResponses)) for _, r := range readResponses { - logger.Infof("Message metadata: %v", r.Metadata) + log.Infof("Message metadata: %v", r.Metadata) assert.Contains(t, string(r.Data), "Integration test message") } } @@ -145,14 +147,14 @@ func testReadIotHubEvents(t *testing.T) { logger := logger.NewLogger("bindings.azure.eventhubs.integration.test") eh := NewAzureEventHubs(logger) err := eh.Init(createIotHubBindingsMetadata()) - assert.Nil(t, err) + require.NoError(t, err) // Invoke az CLI via bash script to send test IoT device events // Requires the AZURE_CREDENTIALS environment variable to be already set (output of `az ad sp create-for-rbac`) cmd := exec.Command("/bin/bash", "../../../tests/scripts/send-iot-device-events.sh") cmd.Env = append(os.Environ(), fmt.Sprintf("IOT_HUB_NAME=%s", os.Getenv(iotHubNameEnvKey))) out, err := cmd.CombinedOutput() - assert.Nil(t, err, "Error in send-iot-device-events.sh:\n%s", out) + assert.NoError(t, err, "Error in send-iot-device-events.sh:\n%s", out) // Setup Read binding to capture readResponses in a closure so that test asserts can be // performed on the main thread, including the case where the handler is never invoked. @@ -177,14 +179,14 @@ func testReadIotHubEvents(t *testing.T) { // Verify expected IoT Hub device event metadata exists // TODO: add device messages than can populate the sysPropPartitionKey and sysPropIotHubConnectionModuleID metadata - assert.Contains(t, r.Metadata, sysPropSequenceNumber, "IoT device event missing: %s", sysPropSequenceNumber) - assert.Contains(t, r.Metadata, sysPropEnqueuedTime, "IoT device event missing: %s", sysPropEnqueuedTime) - assert.Contains(t, r.Metadata, sysPropOffset, "IoT device event missing: %s", sysPropOffset) - assert.Contains(t, r.Metadata, sysPropIotHubDeviceConnectionID, "IoT device event missing: %s", sysPropIotHubDeviceConnectionID) - assert.Contains(t, r.Metadata, sysPropIotHubAuthGenerationID, "IoT device event missing: %s", sysPropIotHubAuthGenerationID) - assert.Contains(t, r.Metadata, sysPropIotHubConnectionAuthMethod, "IoT device event missing: %s", sysPropIotHubConnectionAuthMethod) - assert.Contains(t, r.Metadata, sysPropIotHubEnqueuedTime, "IoT device event missing: %s", sysPropIotHubEnqueuedTime) - assert.Contains(t, r.Metadata, sysPropMessageID, "IoT device event missing: %s", sysPropMessageID) + assert.Contains(t, r.Metadata, "x-opt-sequence-number", "IoT device event missing: x-opt-sequence-number") + assert.Contains(t, r.Metadata, "x-opt-enqueued-time", "IoT device event missing: x-opt-enqueued-time") + assert.Contains(t, r.Metadata, "x-opt-offset", "IoT device event missing: x-opt-offset") + assert.Contains(t, r.Metadata, "iothub-connection-device-id", "IoT device event missing: iothub-connection-device-id") + assert.Contains(t, r.Metadata, "iothub-connection-auth-generation-id", "IoT device event missing: iothub-connection-auth-generation-id") + assert.Contains(t, r.Metadata, "iothub-connection-auth-method", "IoT device event missing: iothub-connection-auth-method") + assert.Contains(t, r.Metadata, "iothub-enqueuedtime", "IoT device event missing: iothub-enqueuedtime") + assert.Contains(t, r.Metadata, "message-id", "IoT device event missing: message-id") } cancel() diff --git a/bindings/azure/eventhubs/eventhubs_test.go b/bindings/azure/eventhubs/eventhubs_test.go deleted file mode 100644 index df5fe7d65..000000000 --- a/bindings/azure/eventhubs/eventhubs_test.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2021 The Dapr Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package eventhubs - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/dapr/components-contrib/bindings" - "github.com/dapr/components-contrib/metadata" - "github.com/dapr/kit/logger" -) - -var testLogger = logger.NewLogger("test") - -func TestGetStoragePrefixString(t *testing.T) { - props := map[string]string{"storageAccountName": "fake", "storageAccountKey": "fake", "consumerGroup": "default", "storageContainerName": "test", "eventHub": "hubName", "eventHubNamespace": "fake"} - - metadata := bindings.Metadata{Base: metadata.Base{Properties: props}} - m, err := parseMetadata(metadata) - - require.NoError(t, err) - - aeh := &AzureEventHubs{logger: testLogger, metadata: m} - - actual, _ := aeh.getStoragePrefixString() - - assert.Equal(t, "dapr-hubName-default-", actual) -} - -func TestGetStoragePrefixStringWithHubNameFromConnectionString(t *testing.T) { - connectionString := "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=fakeKey;SharedAccessKey=key;EntityPath=hubName" - props := map[string]string{"storageAccountName": "fake", "storageAccountKey": "fake", "consumerGroup": "default", "storageContainerName": "test", "connectionString": connectionString} - - metadata := bindings.Metadata{Base: metadata.Base{Properties: props}} - m, err := parseMetadata(metadata) - - require.NoError(t, err) - - aeh := &AzureEventHubs{logger: testLogger, metadata: m} - - actual, _ := aeh.getStoragePrefixString() - - assert.Equal(t, "dapr-hubName-default-", actual) -} - -func TestParseMetadata(t *testing.T) { - t.Run("test valid configuration", func(t *testing.T) { - props := map[string]string{connectionString: "fake", consumerGroup: "mygroup", storageAccountName: "account", storageAccountKey: "key", storageContainerName: "container"} - - bindingsMetadata := bindings.Metadata{Base: metadata.Base{Properties: props}} - - m, err := parseMetadata(bindingsMetadata) - - assert.NoError(t, err) - assert.Equal(t, m.connectionString, "fake") - assert.Equal(t, m.storageAccountName, "account") - assert.Equal(t, m.storageAccountKey, "key") - assert.Equal(t, m.storageContainerName, "container") - assert.Equal(t, m.consumerGroup, "mygroup") - }) - - type invalidConfigTestCase struct { - name string - config map[string]string - errMsg string - } - invalidConfigTestCases := []invalidConfigTestCase{ - { - "missing consumerGroup", - map[string]string{connectionString: "fake", storageAccountName: "account", storageAccountKey: "key", storageContainerName: "container"}, - missingConsumerGroupErrorMsg, - }, - { - "no connectionString requires AAD specific params", - map[string]string{consumerGroup: "fake", storageAccountName: "account", storageAccountKey: "key", storageContainerName: "container"}, - missingHubNameErrorMsg, - }, - { - "only some required params for AAD specified", - map[string]string{consumerGroup: "fakeConsumer", storageAccountName: "account", storageAccountKey: "key", hubName: "namespace", storageContainerName: "fakeContainer"}, - missingHubNamespaceErrorMsg, - }, - { - "missing storageAccountName", - map[string]string{consumerGroup: "fake", connectionString: "fake", storageAccountKey: "key", storageContainerName: "container"}, - missingStorageAccountNameErrorMsg, - }, - { - "missing storageAccountKey", - map[string]string{consumerGroup: "fake", connectionString: "fake", storageAccountName: "name", storageContainerName: "container"}, - missingStorageAccountKeyErrorMsg, - }, - { - "missing storageContainerName", - map[string]string{consumerGroup: "fake", connectionString: "fake", storageAccountName: "name", storageAccountKey: "key"}, - missingStorageContainerNameErrorMsg, - }, - } - - for _, c := range invalidConfigTestCases { - t.Run(c.name, func(t *testing.T) { - bindingsMetadata := bindings.Metadata{Base: metadata.Base{Properties: c.config}} - _, err := parseMetadata(bindingsMetadata) - assert.Error(t, err) - assert.Equal(t, err.Error(), c.errMsg) - }) - } -} diff --git a/internal/component/azure/eventhubs/eventhubs.go b/internal/component/azure/eventhubs/eventhubs.go index df7900117..ce591bb33 100644 --- a/internal/component/azure/eventhubs/eventhubs.go +++ b/internal/component/azure/eventhubs/eventhubs.go @@ -38,8 +38,9 @@ import ( // AzureEventHubs allows sending/receiving Azure Event Hubs events. // This is an abstract class used by both the pubsub and binding components. type AzureEventHubs struct { - metadata *azureEventHubsMetadata - logger logger.Logger + metadata *azureEventHubsMetadata + logger logger.Logger + isBinding bool backOffConfig retry.Config producersLock *sync.RWMutex @@ -54,9 +55,10 @@ type AzureEventHubs struct { } // NewAzureEventHubs returns a new Azure Event hubs instance. -func NewAzureEventHubs(logger logger.Logger) *AzureEventHubs { +func NewAzureEventHubs(logger logger.Logger, isBinding bool) *AzureEventHubs { return &AzureEventHubs{ logger: logger, + isBinding: isBinding, producersLock: &sync.RWMutex{}, producers: make(map[string]*azeventhubs.ProducerClient, 1), checkpointStoreLock: &sync.RWMutex{}, @@ -66,24 +68,14 @@ func NewAzureEventHubs(logger logger.Logger) *AzureEventHubs { // Init connects to Azure Event Hubs. func (aeh *AzureEventHubs) Init(metadata map[string]string) error { - m, err := parseEventHubsMetadata(metadata, aeh.logger) + m, err := parseEventHubsMetadata(metadata, aeh.isBinding, aeh.logger) if err != nil { return err } aeh.metadata = m - if aeh.metadata.ConnectionString != "" { - // Connect using the connection string - hubName := hubNameFromConnString(aeh.metadata.ConnectionString) - if hubName != "" { - aeh.logger.Infof(`The provided connection string is specific to the Event Hub ("entity path") '%s'; publishing or subscribing to a topic that does not match this Event Hub will fail when attempted`, hubName) - } else { - aeh.logger.Infof(`The provided connection string does not contain an Event Hub name ("entity path"); the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored`) - } - - aeh.metadata.hubName = hubName - } else { - // Connect via Azure AD + if aeh.metadata.ConnectionString == "" { + // If connecting via Azure AD, we need to do some more initialization var env azauth.EnvironmentSettings env, err = azauth.NewEnvironmentSettings("eventhubs", metadata) if err != nil { @@ -116,6 +108,12 @@ func (aeh *AzureEventHubs) Init(metadata map[string]string) error { return nil } +// EventHubName returns the parsed eventHub property from the metadata. +// It's used by the binding only. +func (aeh *AzureEventHubs) EventHubName() string { + return aeh.metadata.hubName +} + // Publish a batch of messages. func (aeh *AzureEventHubs) Publish(ctx context.Context, topic string, messages []*azeventhubs.EventData, batchOpts *azeventhubs.EventDataBatchOptions) error { // Get the producer client @@ -147,7 +145,7 @@ func (aeh *AzureEventHubs) Publish(ctx context.Context, topic string, messages [ return nil } -// Subscribe receives data from Azure Event Hubs. +// Subscribe receives data from Azure Event Hubs in background. func (aeh *AzureEventHubs) Subscribe(subscribeCtx context.Context, topic string, getAllProperties bool, handler SubscribeHandler) (err error) { if aeh.metadata.ConsumerGroup == "" { return errors.New("property consumerID is required to subscribe to an Event Hub topic") @@ -613,13 +611,17 @@ func (aeh *AzureEventHubs) createStorageClient() (*azblob.Client, error) { } } else { // Use Azure AD - settings, err := azauth.NewEnvironmentSettings("storage", aeh.metadata.properties) + var ( + settings azauth.EnvironmentSettings + credential azcore.TokenCredential + ) + settings, err = azauth.NewEnvironmentSettings("storage", aeh.metadata.properties) if err != nil { - return nil, err + return nil, fmt.Errorf("error getting Azure environment settings: %w", err) } - credential, tokenErr := settings.GetTokenCredential() - if tokenErr != nil { - return nil, fmt.Errorf("invalid Azure Storage token credentials with error: %w", tokenErr) + credential, err = settings.GetTokenCredential() + if err != nil { + return nil, fmt.Errorf("invalid Azure Storage token credentials with error: %w", err) } client, err = azblob.NewClient(accountURL, credential, &options) if err != nil { diff --git a/internal/component/azure/eventhubs/eventhubs_test.go b/internal/component/azure/eventhubs/eventhubs_test.go index b84167255..e3d303ff1 100644 --- a/internal/component/azure/eventhubs/eventhubs_test.go +++ b/internal/component/azure/eventhubs/eventhubs_test.go @@ -28,7 +28,7 @@ func TestParseEventHubsMetadata(t *testing.T) { t.Run("test valid connectionString configuration", func(t *testing.T) { metadata := map[string]string{"connectionString": "fake"} - m, err := parseEventHubsMetadata(metadata, testLogger) + m, err := parseEventHubsMetadata(metadata, false, testLogger) require.NoError(t, err) assert.Equal(t, "fake", m.ConnectionString) @@ -37,7 +37,7 @@ func TestParseEventHubsMetadata(t *testing.T) { t.Run("test namespace given", func(t *testing.T) { metadata := map[string]string{"eventHubNamespace": "fake.servicebus.windows.net"} - m, err := parseEventHubsMetadata(metadata, testLogger) + m, err := parseEventHubsMetadata(metadata, false, testLogger) require.NoError(t, err) assert.Equal(t, "fake.servicebus.windows.net", m.EventHubNamespace) @@ -46,7 +46,7 @@ func TestParseEventHubsMetadata(t *testing.T) { t.Run("test namespace adds FQDN", func(t *testing.T) { metadata := map[string]string{"eventHubNamespace": "fake"} - m, err := parseEventHubsMetadata(metadata, testLogger) + m, err := parseEventHubsMetadata(metadata, false, testLogger) require.NoError(t, err) assert.Equal(t, "fake.servicebus.windows.net", m.EventHubNamespace) @@ -58,7 +58,7 @@ func TestParseEventHubsMetadata(t *testing.T) { "eventHubNamespace": "fake", } - _, err := parseEventHubsMetadata(metadata, testLogger) + _, err := parseEventHubsMetadata(metadata, false, testLogger) require.Error(t, err) assert.ErrorContains(t, err, "only one of connectionString or eventHubNamespace should be passed") @@ -67,7 +67,7 @@ func TestParseEventHubsMetadata(t *testing.T) { t.Run("test missing metadata", func(t *testing.T) { metadata := map[string]string{} - _, err := parseEventHubsMetadata(metadata, testLogger) + _, err := parseEventHubsMetadata(metadata, false, testLogger) require.Error(t, err) assert.ErrorContains(t, err, "one of connectionString or eventHubNamespace is required") diff --git a/internal/component/azure/eventhubs/metadata.go b/internal/component/azure/eventhubs/metadata.go index 7eb1ac5ab..1bf5a1f0f 100644 --- a/internal/component/azure/eventhubs/metadata.go +++ b/internal/component/azure/eventhubs/metadata.go @@ -39,6 +39,9 @@ type azureEventHubsMetadata struct { SubscriptionID string `json:"subscriptionID" mapstructure:"subscriptionID"` ResourceGroupName string `json:"resourceGroupName" mapstructure:"resourceGroupName"` + // Binding only + EventHub string `json:"eventHub" mapstructure:"eventHub"` + // Internal properties namespaceName string hubName string @@ -46,7 +49,7 @@ type azureEventHubsMetadata struct { properties map[string]string } -func parseEventHubsMetadata(meta map[string]string, log logger.Logger) (*azureEventHubsMetadata, error) { +func parseEventHubsMetadata(meta map[string]string, isBinding bool, log logger.Logger) (*azureEventHubsMetadata, error) { var m azureEventHubsMetadata err := metadata.DecodeMetadata(meta, &m) if err != nil { @@ -64,6 +67,40 @@ func parseEventHubsMetadata(meta map[string]string, log logger.Logger) (*azureEv return nil, errors.New("only one of connectionString or eventHubNamespace should be passed") } + // For the binding, we need to have a property "eventHub" which is the topic name unless it's included in the connection string + if isBinding { + if m.ConnectionString == "" { + if m.EventHub == "" { + return nil, errors.New("property eventHub is required when connecting with a connection string") + } + m.hubName = m.EventHub + } else { + hubName := hubNameFromConnString(m.ConnectionString) + if hubName != "" { + m.hubName = hubName + } else if m.EventHub != "" { + m.hubName = m.EventHub + } else { + return nil, errors.New("the provided connection string does not contain a value for 'EntityPath' and no eventHub property was passed") + } + } + } else { + // Ignored when not a binding + m.EventHub = "" + + // If connecting using a connection string, parse hubName + if m.ConnectionString != "" { + hubName := hubNameFromConnString(m.ConnectionString) + if hubName != "" { + log.Infof(`The provided connection string is specific to the Event Hub ("entity path") '%s'; publishing or subscribing to a topic that does not match this Event Hub will fail when attempted`, hubName) + } else { + log.Infof(`The provided connection string does not contain an Event Hub name ("entity path"); the connection will be established on first publish/subscribe and req.Topic field in incoming requests will be honored`) + } + + m.hubName = hubName + } + } + // If both storageConnectionString and storageAccountKey are specified, show a warning because the connection string will take priority if m.StorageConnectionString != "" && m.StorageAccountName != "" { log.Warn("Property storageAccountKey is ignored when storageConnectionString is present") diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 0596a662e..195de1715 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -36,7 +36,7 @@ type AzureEventHubs struct { // NewAzureEventHubs returns a new Azure Event hubs instance. func NewAzureEventHubs(logger logger.Logger) pubsub.PubSub { return &AzureEventHubs{ - AzureEventHubs: impl.NewAzureEventHubs(logger), + AzureEventHubs: impl.NewAzureEventHubs(logger, false), } } @@ -130,6 +130,7 @@ func (aeh *AzureEventHubs) Subscribe(ctx context.Context, req pubsub.SubscribeRe getAllProperties := utils.IsTruthy(req.Metadata["requireAllProperties"]) // Start the subscription + // This is non-blocking return aeh.AzureEventHubs.Subscribe(ctx, topic, getAllProperties, func(ctx context.Context, data []byte, metadata map[string]string) error { res := pubsub.NewMessage{ Data: data, diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/iothub/eventhubs.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/iothub/eventhubs.yaml index d87b88570..9cf7ac4e1 100644 --- a/tests/certification/bindings/azure/eventhubs/components/binding/iothub/eventhubs.yaml +++ b/tests/certification/bindings/azure/eventhubs/components/binding/iothub/eventhubs.yaml @@ -29,7 +29,5 @@ spec: secretKeyRef: name: AzureEventHubsBindingsContainer key: AzureEventHubsBindingsContainer - - name: PartitionID - value: 0 auth: secretStore: envvar-secret-store diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml index aff4c68b3..bf98f0b01 100644 --- a/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml +++ b/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml @@ -51,7 +51,5 @@ spec: secretKeyRef: name: AzureEventHubsBindingsContainer value: AzureEventHubsBindingsContainer - - name: partitionID - value: 0 auth: secretStore: envvar-secret-store diff --git a/tests/conformance/bindings/bindings.go b/tests/conformance/bindings/bindings.go index c98c3cd78..4bb143302 100644 --- a/tests/conformance/bindings/bindings.go +++ b/tests/conformance/bindings/bindings.go @@ -202,7 +202,7 @@ func ConformanceTests(t *testing.T, props map[string]string, inputBinding bindin return nil, nil }) - assert.True(t, err == nil || errors.Is(err, context.Canceled), "expected Read canceled on Close") + assert.Truef(t, err == nil || errors.Is(err, context.Canceled), "expected Read canceled on Close, got: %v", err) }) // Special case for message brokers that are also bindings // Need a small wait here because with brokers like MQTT From 1fd48dd91217530a66d08cb07f51b100f090ac9c Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Sat, 21 Jan 2023 19:40:52 +0000 Subject: [PATCH 32/42] Fixes Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- internal/component/azure/eventhubs/metadata.go | 10 ++++++++-- tests/conformance/bindings/bindings.go | 12 ++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/internal/component/azure/eventhubs/metadata.go b/internal/component/azure/eventhubs/metadata.go index 1bf5a1f0f..26a23ddf3 100644 --- a/internal/component/azure/eventhubs/metadata.go +++ b/internal/component/azure/eventhubs/metadata.go @@ -28,7 +28,7 @@ import ( type azureEventHubsMetadata struct { ConnectionString string `json:"connectionString" mapstructure:"connectionString"` EventHubNamespace string `json:"eventHubNamespace" mapstructure:"eventHubNamespace"` - ConsumerGroup string `json:"consumerID" mapstructure:"consumerID"` + ConsumerID string `json:"consumerID" mapstructure:"consumerID"` StorageConnectionString string `json:"storageConnectionString" mapstructure:"storageConnectionString"` StorageAccountName string `json:"storageAccountName" mapstructure:"storageAccountName"` StorageAccountKey string `json:"storageAccountKey" mapstructure:"storageAccountKey"` @@ -40,7 +40,8 @@ type azureEventHubsMetadata struct { ResourceGroupName string `json:"resourceGroupName" mapstructure:"resourceGroupName"` // Binding only - EventHub string `json:"eventHub" mapstructure:"eventHub"` + EventHub string `json:"eventHub" mapstructure:"eventHub"` + ConsumerGroup string `json:"consumerGroup" mapstructure:"consumerGroup"` // Alias for ConsumerID // Internal properties namespaceName string @@ -67,6 +68,11 @@ func parseEventHubsMetadata(meta map[string]string, isBinding bool, log logger.L return nil, errors.New("only one of connectionString or eventHubNamespace should be passed") } + // ConsumerGroup is an alias for ConsumerID + if m.ConsumerID != "" && m.ConsumerGroup == "" { + m.ConsumerGroup = m.ConsumerID + } + // For the binding, we need to have a property "eventHub" which is the topic name unless it's included in the connection string if isBinding { if m.ConnectionString == "" { diff --git a/tests/conformance/bindings/bindings.go b/tests/conformance/bindings/bindings.go index 4bb143302..0c875d39e 100644 --- a/tests/conformance/bindings/bindings.go +++ b/tests/conformance/bindings/bindings.go @@ -146,23 +146,23 @@ func ConformanceTests(t *testing.T, props map[string]string, inputBinding bindin if config.HasOperation("read") { errInp := bindings.PingInpBinding(inputBinding) // TODO: Ideally, all stable components should implenment ping function, - // so will only assert assert.Nil(t, err) finally, i.e. when current implementation + // so will only assert assert.NoError(t, err) finally, i.e. when current implementation // implements ping in existing stable components if errInp != nil { assert.EqualError(t, errInp, "ping is not implemented by this input binding") } else { - assert.Nil(t, errInp) + assert.NoError(t, errInp) } } if config.HasOperation("operations") { errOut := bindings.PingOutBinding(outputBinding) // TODO: Ideally, all stable components should implenment ping function, - // so will only assert assert.Nil(t, err) finally, i.e. when current implementation + // so will only assert assert.NoError(t, err) finally, i.e. when current implementation // implements ping in existing stable components if errOut != nil { assert.EqualError(t, errOut, "ping is not implemented by this output binding") } else { - assert.Nil(t, errOut) + assert.NoError(t, errOut) } } }) @@ -233,7 +233,7 @@ func ConformanceTests(t *testing.T, props map[string]string, inputBinding bindin req := config.createInvokeRequest() req.Operation = bindings.GetOperation resp, err := outputBinding.Invoke(context.Background(), &req) - assert.Nil(t, err, "expected no error invoking output binding") + assert.NoError(t, err, "expected no error invoking output binding") if createPerformed { assert.Equal(t, req.Data, resp.Data) } @@ -276,7 +276,7 @@ func ConformanceTests(t *testing.T, props map[string]string, inputBinding bindin req := config.createInvokeRequest() req.Operation = bindings.DeleteOperation _, err := outputBinding.Invoke(context.Background(), &req) - assert.Nil(t, err, "expected no error invoking output binding") + assert.NoError(t, err, "expected no error invoking output binding") if createPerformed && config.HasOperation(string(bindings.GetOperation)) { req.Operation = bindings.GetOperation From cc06a37d376c3eedd9da7ce6103dee8caa5cdb45 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Sat, 21 Jan 2023 19:46:09 +0000 Subject: [PATCH 33/42] Enable fallback route in IoT hub to fix tests Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../conformance/azure/conf-test-azure-iothub.bicep | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/infrastructure/conformance/azure/conf-test-azure-iothub.bicep b/.github/infrastructure/conformance/azure/conf-test-azure-iothub.bicep index 231fc7c17..009757491 100644 --- a/.github/infrastructure/conformance/azure/conf-test-azure-iothub.bicep +++ b/.github/infrastructure/conformance/azure/conf-test-azure-iothub.bicep @@ -34,6 +34,15 @@ resource iotHub 'Microsoft.Devices/IotHubs@2021-03-31' = { partitionCount: 2 } } + routing: { + fallbackRoute: { + name: '$fallback' + source: 'DeviceMessages' + condition: 'true' + endpointNames: ['events'] + isEnabled: true + } + } } } From d986d3b6ff110d96f61ce514b70673eb70b55dfa Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Sat, 21 Jan 2023 20:00:45 +0000 Subject: [PATCH 34/42] More fixes Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../component/azure/eventhubs/metadata.go | 8 ++++++++ .../pubsub/azure/eventhubs/eventhubs_test.go | 19 +++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/internal/component/azure/eventhubs/metadata.go b/internal/component/azure/eventhubs/metadata.go index 26a23ddf3..aa56df0b5 100644 --- a/internal/component/azure/eventhubs/metadata.go +++ b/internal/component/azure/eventhubs/metadata.go @@ -42,6 +42,7 @@ type azureEventHubsMetadata struct { // Binding only EventHub string `json:"eventHub" mapstructure:"eventHub"` ConsumerGroup string `json:"consumerGroup" mapstructure:"consumerGroup"` // Alias for ConsumerID + PartitionID string `json:"partitionID" mapstructure:"partitionID"` // Deprecated // Internal properties namespaceName string @@ -90,9 +91,16 @@ func parseEventHubsMetadata(meta map[string]string, isBinding bool, log logger.L return nil, errors.New("the provided connection string does not contain a value for 'EntityPath' and no eventHub property was passed") } } + + // Property partitionID is deprecated + if m.PartitionID != "" { + log.Info("Property partitionID is deprecated and will be ignored") + m.PartitionID = "" + } } else { // Ignored when not a binding m.EventHub = "" + m.PartitionID = "" // If connecting using a connection string, parse hubName if m.ConnectionString != "" { diff --git a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go index 83a596394..9bead326f 100644 --- a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go +++ b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go @@ -169,7 +169,13 @@ func TestEventhubs(t *testing.T) { } } - deleteEventhub := func(ctx flow.Context) error { + flowDoneCh := make(chan struct{}, 1) + flowDone := func(ctx flow.Context) error { + close(flowDoneCh) + return nil + } + + deleteEventhub := func() error { output, err := exec.Command("/bin/sh", "delete-eventhub.sh", topicToBeCreated).Output() assert.NoErrorf(t, err, "Error in delete-eventhub.sh.:\n%s", string(output)) return nil @@ -295,10 +301,15 @@ func TestEventhubs(t *testing.T) { Step("add expected IoT messages (simulate add message to IoT Hub)", publishMessageAsDevice(consumerGroup5)). Step("verify if app5 has recevied messages published to IoT topic", assertMessages(10*time.Second, consumerGroup5)). - // cleanup azure assets created as part of tests - Step("wait", flow.Sleep(5*time.Second)). - Step("delete eventhub created as part of the eventhub management test", deleteEventhub). + // Run the flow + Step("mark as complete", flowDone). Run() + + // Cleanup Azure assets created as part of tests + <-flowDoneCh + time.Sleep(5 * time.Second) + fmt.Println("Deleting EventHub resources created as part of the management test…") + deleteEventhub() } func componentRuntimeOptions(instance int) []runtime.Option { From 29ab4242e93b4650be0835c43b627ab5f5865aae Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Sat, 21 Jan 2023 20:34:44 +0000 Subject: [PATCH 35/42] Removed unused code Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- go.mod | 4 -- go.sum | 9 ---- internal/authentication/azure/storage.go | 45 ------------------- .../bindings/azure/blobstorage/go.mod | 1 - .../bindings/azure/blobstorage/go.sum | 10 ----- .../bindings/azure/cosmosdb/go.mod | 1 - .../bindings/azure/cosmosdb/go.sum | 10 ----- .../bindings/azure/eventhubs/go.mod | 11 ++--- .../bindings/azure/eventhubs/go.sum | 32 +++---------- .../bindings/azure/servicebusqueues/go.mod | 1 - .../bindings/azure/servicebusqueues/go.sum | 10 ----- .../bindings/azure/storagequeues/go.mod | 1 - .../bindings/azure/storagequeues/go.sum | 10 ----- .../pubsub/azure/eventhubs/go.mod | 1 - .../pubsub/azure/eventhubs/go.sum | 10 ----- .../pubsub/azure/servicebus/topics/go.mod | 1 - .../pubsub/azure/servicebus/topics/go.sum | 10 ----- .../secretstores/azure/keyvault/go.mod | 1 - .../secretstores/azure/keyvault/go.sum | 10 ----- .../state/azure/blobstorage/go.mod | 1 - .../state/azure/blobstorage/go.sum | 10 ----- .../certification/state/azure/cosmosdb/go.mod | 1 - .../certification/state/azure/cosmosdb/go.sum | 10 ----- .../state/azure/tablestorage/go.mod | 1 - .../state/azure/tablestorage/go.sum | 10 ----- 25 files changed, 10 insertions(+), 201 deletions(-) diff --git a/go.mod b/go.mod index 33d1acf27..2bbdd8ea8 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( cloud.google.com/go/storage v1.28.1 dubbo.apache.org/dubbo-go/v3 v3.0.3-0.20230118042253-4f159a2b38f3 github.com/Azure/azure-amqp-common-go/v4 v4.0.0 - github.com/Azure/azure-event-hubs-go/v3 v3.4.0 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 @@ -21,7 +20,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 - github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd github.com/Azure/go-amqp v0.18.0 github.com/Azure/go-autorest/autorest v0.11.28 @@ -187,7 +185,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/deepmap/oapi-codegen v1.3.6 // indirect - github.com/devigned/tab v0.1.1 // indirect github.com/dghubble/sling v1.4.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -262,7 +259,6 @@ require ( github.com/jinzhu/copier v0.3.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/jpillora/backoff v1.0.0 // indirect github.com/k0kubun/pp v3.0.1+incompatible // indirect github.com/kataras/go-errors v0.0.3 // indirect github.com/kataras/go-serializer v0.0.4 // indirect diff --git a/go.sum b/go.sum index 5b2490708..c2cc18c41 100644 --- a/go.sum +++ b/go.sum @@ -406,8 +406,6 @@ github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA= github.com/Azure/azure-amqp-common-go/v4 v4.0.0 h1:mV5O74KYmonn0ZXtwfMjGUtZ9Z+Hv7AIFVS1s03sRvo= github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8ONkEwRgWXqes3SUt1Ftrc= -github.com/Azure/azure-event-hubs-go/v3 v3.4.0 h1:LtH0nHkXivyV/GajOu5ZFC5sb/5KZ8j+9U8UsfHVTOo= -github.com/Azure/azure-event-hubs-go/v3 v3.4.0/go.mod h1:ODgt5C1/c73FToYj+mWojUJLXF877ALc6G4XnfRFlAY= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= @@ -440,8 +438,6 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0/go.mod h1:Y3gnVwfaz8h6L1YHar+NfWORtBoVUSB5h4GlGkdeF7Q= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-amqp v0.18.0 h1:95bTiJq0oxjK1RUlt5T3HF/THj6jWTRZpSXMPSOJLz8= @@ -451,7 +447,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -730,8 +725,6 @@ github.com/deepmap/oapi-codegen v1.3.6 h1:Wj44p9A0V0PJ+AUg0BWdyGcsS1LY18U+0rCuPQ github.com/deepmap/oapi-codegen v1.3.6/go.mod h1:aBozjEveG+33xPiP55Iw/XbVkhtZHEGLq3nxlX0+hfU= github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= -github.com/devigned/tab v0.1.1 h1:3mD6Kb1mUOYeLpJvTVSDwSg5ZsfSxfvxGRTxRsJsITA= -github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dghubble/go-twitter v0.0.0-20221104224141-912508c3888b h1:XQu6o3AwJx/jsg9LZ41uIeUdXK5be099XFfFn6H9ikk= github.com/dghubble/go-twitter v0.0.0-20221104224141-912508c3888b/go.mod h1:B0/qdW5XUupJvcsx40hnVbfjzz9He5YpYXx6eVVdiSY= github.com/dghubble/oauth1 v0.7.2 h1:pwcinOZy8z6XkNxvPmUDY52M7RDPxt0Xw1zgZ6Cl5JA= @@ -1047,7 +1040,6 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= @@ -1249,7 +1241,6 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= diff --git a/internal/authentication/azure/storage.go b/internal/authentication/azure/storage.go index 191609775..e8ae78f4e 100644 --- a/internal/authentication/azure/storage.go +++ b/internal/authentication/azure/storage.go @@ -17,7 +17,6 @@ import ( "fmt" "time" - "github.com/Azure/azure-storage-blob-go/azblob" "github.com/Azure/azure-storage-queue-go/azqueue" "github.com/Azure/go-autorest/autorest/azure" @@ -34,50 +33,6 @@ var ( StorageEndpointKeys = []string{"endpoint", "storageEndpoint", "storageAccountEndpoint", "queueEndpointUrl"} ) -// GetAzureStorageBlobCredentials returns a azblob.Credential object that can be used to authenticate an Azure Blob Storage SDK pipeline ("track 1"). -// First it tries to authenticate using shared key credentials (using an account key) if present. It falls back to attempting to use Azure AD (via a service principal or MSI). -func GetAzureStorageBlobCredentials(log logger.Logger, accountName string, metadata map[string]string) (azblob.Credential, *azure.Environment, error) { - settings, err := NewEnvironmentSettings("storage", metadata) - if err != nil { - return nil, nil, err - } - - // Try using shared key credentials first - accountKey, ok := mdutils.GetMetadataProperty(metadata, StorageAccountKeyKeys...) - if ok && accountKey != "" { - credential, newSharedKeyErr := azblob.NewSharedKeyCredential(accountName, accountKey) - if err != nil { - return nil, nil, fmt.Errorf("invalid credentials with error: %s", newSharedKeyErr.Error()) - } - - return credential, settings.AzureEnvironment, nil - } - - // Fallback to using Azure AD - spt, err := settings.GetServicePrincipalToken() - if err != nil { - return nil, nil, err - } - var tokenRefresher azblob.TokenRefresher = func(credential azblob.TokenCredential) time.Duration { - log.Debug("Refreshing Azure Storage auth token") - err := spt.Refresh() - if err != nil { - panic(err) - } - token := spt.Token() - credential.SetToken(token.AccessToken) - - // Make the token expire 2 minutes earlier to get some extra buffer - exp := token.Expires().Sub(time.Now().Add(2 * time.Minute)) - log.Debug("Received new token, valid for", exp) - - return exp - } - credential := azblob.NewTokenCredential("", tokenRefresher) - - return credential, settings.AzureEnvironment, nil -} - // GetAzureStorageQueueCredentials returns a azqueues.Credential object that can be used to authenticate an Azure Queue Storage SDK pipeline ("track 1"). // First it tries to authenticate using shared key credentials (using an account key) if present. It falls back to attempting to use Azure AD (via a service principal or MSI). func GetAzureStorageQueueCredentials(log logger.Logger, accountName string, metadata map[string]string) (azqueue.Credential, *azure.Environment, error) { diff --git a/tests/certification/bindings/azure/blobstorage/go.mod b/tests/certification/bindings/azure/blobstorage/go.mod index d05e8426f..b1a1cccf6 100644 --- a/tests/certification/bindings/azure/blobstorage/go.mod +++ b/tests/certification/bindings/azure/blobstorage/go.mod @@ -20,7 +20,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect diff --git a/tests/certification/bindings/azure/blobstorage/go.sum b/tests/certification/bindings/azure/blobstorage/go.sum index aceab4e19..765ba19e7 100644 --- a/tests/certification/bindings/azure/blobstorage/go.sum +++ b/tests/certification/bindings/azure/blobstorage/go.sum @@ -48,8 +48,6 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSq github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -57,7 +55,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -162,7 +159,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -275,7 +271,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -364,7 +359,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -575,8 +569,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -654,7 +646,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -923,7 +914,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/bindings/azure/cosmosdb/go.mod b/tests/certification/bindings/azure/cosmosdb/go.mod index 3bde702fd..8c9e1aa36 100644 --- a/tests/certification/bindings/azure/cosmosdb/go.mod +++ b/tests/certification/bindings/azure/cosmosdb/go.mod @@ -23,7 +23,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect diff --git a/tests/certification/bindings/azure/cosmosdb/go.sum b/tests/certification/bindings/azure/cosmosdb/go.sum index 8804a0c49..5b1dff267 100644 --- a/tests/certification/bindings/azure/cosmosdb/go.sum +++ b/tests/certification/bindings/azure/cosmosdb/go.sum @@ -50,8 +50,6 @@ github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3 h1:x1shk+tVZ6kLwIQMn4 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3/go.mod h1:Fy3bbChFm4cZn6oIxYYqKB2FG3rBDxk3NZDLDJCHl+Q= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -59,7 +57,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -166,7 +163,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -279,7 +275,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -369,7 +364,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -580,8 +574,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -659,7 +651,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -928,7 +919,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/bindings/azure/eventhubs/go.mod b/tests/certification/bindings/azure/eventhubs/go.mod index 17769b4a4..8e78a324f 100644 --- a/tests/certification/bindings/azure/eventhubs/go.mod +++ b/tests/certification/bindings/azure/eventhubs/go.mod @@ -17,23 +17,20 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/Azure/azure-amqp-common-go/v4 v4.0.0 // indirect - github.com/Azure/azure-event-hubs-go/v3 v3.4.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect - github.com/Azure/go-amqp v0.18.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect - github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect @@ -47,7 +44,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/devigned/tab v0.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect @@ -87,7 +83,6 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/jhump/protoreflect v1.13.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.12 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/tests/certification/bindings/azure/eventhubs/go.sum b/tests/certification/bindings/azure/eventhubs/go.sum index 3795c4e7b..f4373c5d8 100644 --- a/tests/certification/bindings/azure/eventhubs/go.sum +++ b/tests/certification/bindings/azure/eventhubs/go.sum @@ -37,31 +37,28 @@ github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a h1: github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a/go.mod h1:C0A1KeiVHs+trY6gUTPhhGammbrZ30ZfXRW/nuT7HLw= github.com/Azure/azure-amqp-common-go/v4 v4.0.0 h1:mV5O74KYmonn0ZXtwfMjGUtZ9Z+Hv7AIFVS1s03sRvo= github.com/Azure/azure-amqp-common-go/v4 v4.0.0/go.mod h1:4+qRvizIo4+CbGG552O6a8ONkEwRgWXqes3SUt1Ftrc= -github.com/Azure/azure-event-hubs-go/v3 v3.4.0 h1:LtH0nHkXivyV/GajOu5ZFC5sb/5KZ8j+9U8UsfHVTOo= -github.com/Azure/azure-event-hubs-go/v3 v3.4.0/go.mod h1:ODgt5C1/c73FToYj+mWojUJLXF877ALc6G4XnfRFlAY= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 h1:X/ePaAG8guM7j5WORT5eEIw7cGUxe9Ah1jEQJKLmmSo= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0/go.mod h1:5dog28UP3dd1BnCPFYvyHfsmA+Phmoezt+KWT5cZnyc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 h1:BWeAAEzkCnL0ABVJqs+4mYudNch7oFGPtTlSmIWL8ms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0/go.mod h1:Y3gnVwfaz8h6L1YHar+NfWORtBoVUSB5h4GlGkdeF7Q= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= -github.com/Azure/go-amqp v0.18.0 h1:95bTiJq0oxjK1RUlt5T3HF/THj6jWTRZpSXMPSOJLz8= -github.com/Azure/go-amqp v0.18.0/go.mod h1:+bg0x3ce5+Q3ahCEXnCsGG3ETpDQe3MEVnOuT2ywPwc= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -74,10 +71,6 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= -github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= -github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= @@ -146,8 +139,6 @@ github.com/dapr/kit v0.0.4-0.20230105202559-fcb09958bfb0/go.mod h1:RFN6r5pZzhrel github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devigned/tab v0.1.1 h1:3mD6Kb1mUOYeLpJvTVSDwSg5ZsfSxfvxGRTxRsJsITA= -github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= @@ -172,8 +163,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -286,7 +275,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -354,10 +342,9 @@ github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7 github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.13.0 h1:zrrZqa7JAc2YGgPSzZZkmUXJ5G6NRPdxOg/9t7ISImA= github.com/jhump/protoreflect v1.13.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -379,7 +366,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -595,8 +581,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -674,7 +658,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -943,7 +926,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/bindings/azure/servicebusqueues/go.mod b/tests/certification/bindings/azure/servicebusqueues/go.mod index d4518fc39..b79e61c80 100644 --- a/tests/certification/bindings/azure/servicebusqueues/go.mod +++ b/tests/certification/bindings/azure/servicebusqueues/go.mod @@ -21,7 +21,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-amqp v0.18.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/bindings/azure/servicebusqueues/go.sum b/tests/certification/bindings/azure/servicebusqueues/go.sum index b9d4b2c06..c8e166128 100644 --- a/tests/certification/bindings/azure/servicebusqueues/go.sum +++ b/tests/certification/bindings/azure/servicebusqueues/go.sum @@ -48,8 +48,6 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSq github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4 h1:kaZamwZwmUqnECvnPkf1LBRBIFYYCy3E0gKHn/UFSD0= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4/go.mod h1:uDLwkzCJMvTrHsvtiVFeAp85hi3W77zvs61wrpc+6ho= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-amqp v0.18.0 h1:95bTiJq0oxjK1RUlt5T3HF/THj6jWTRZpSXMPSOJLz8= @@ -59,7 +57,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -166,7 +163,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -280,7 +276,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -372,7 +367,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -590,8 +584,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -669,7 +661,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -938,7 +929,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/bindings/azure/storagequeues/go.mod b/tests/certification/bindings/azure/storagequeues/go.mod index 422b576eb..6c2e211d1 100644 --- a/tests/certification/bindings/azure/storagequeues/go.mod +++ b/tests/certification/bindings/azure/storagequeues/go.mod @@ -20,7 +20,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect diff --git a/tests/certification/bindings/azure/storagequeues/go.sum b/tests/certification/bindings/azure/storagequeues/go.sum index f448eb77d..015dbd2a8 100644 --- a/tests/certification/bindings/azure/storagequeues/go.sum +++ b/tests/certification/bindings/azure/storagequeues/go.sum @@ -46,8 +46,6 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfR github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -55,7 +53,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -160,7 +157,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -273,7 +269,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -364,7 +359,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -580,8 +574,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -659,7 +651,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -928,7 +919,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/pubsub/azure/eventhubs/go.mod b/tests/certification/pubsub/azure/eventhubs/go.mod index 74dd78093..ca7ee5c3a 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.mod +++ b/tests/certification/pubsub/azure/eventhubs/go.mod @@ -24,7 +24,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v0.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect diff --git a/tests/certification/pubsub/azure/eventhubs/go.sum b/tests/certification/pubsub/azure/eventhubs/go.sum index a27a814f2..bfbd18f8a 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.sum +++ b/tests/certification/pubsub/azure/eventhubs/go.sum @@ -52,8 +52,6 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.0.0/go.mod h1:Y3gnVwfaz8h6L1YHar+NfWORtBoVUSB5h4GlGkdeF7Q= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -61,7 +59,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -166,7 +163,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -279,7 +275,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -371,7 +366,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -585,8 +579,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -664,7 +656,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -933,7 +924,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/pubsub/azure/servicebus/topics/go.mod b/tests/certification/pubsub/azure/servicebus/topics/go.mod index 88e9de803..c9bc65549 100644 --- a/tests/certification/pubsub/azure/servicebus/topics/go.mod +++ b/tests/certification/pubsub/azure/servicebus/topics/go.mod @@ -22,7 +22,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-amqp v0.18.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/pubsub/azure/servicebus/topics/go.sum b/tests/certification/pubsub/azure/servicebus/topics/go.sum index b9d4b2c06..c8e166128 100644 --- a/tests/certification/pubsub/azure/servicebus/topics/go.sum +++ b/tests/certification/pubsub/azure/servicebus/topics/go.sum @@ -48,8 +48,6 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSq github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4 h1:kaZamwZwmUqnECvnPkf1LBRBIFYYCy3E0gKHn/UFSD0= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.4/go.mod h1:uDLwkzCJMvTrHsvtiVFeAp85hi3W77zvs61wrpc+6ho= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-amqp v0.18.0 h1:95bTiJq0oxjK1RUlt5T3HF/THj6jWTRZpSXMPSOJLz8= @@ -59,7 +57,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -166,7 +163,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -280,7 +276,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -372,7 +367,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -590,8 +584,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -669,7 +661,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -938,7 +929,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/secretstores/azure/keyvault/go.mod b/tests/certification/secretstores/azure/keyvault/go.mod index 8c7eaf6ab..c04087af4 100644 --- a/tests/certification/secretstores/azure/keyvault/go.mod +++ b/tests/certification/secretstores/azure/keyvault/go.mod @@ -21,7 +21,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect diff --git a/tests/certification/secretstores/azure/keyvault/go.sum b/tests/certification/secretstores/azure/keyvault/go.sum index 75326a28c..9043fbe80 100644 --- a/tests/certification/secretstores/azure/keyvault/go.sum +++ b/tests/certification/secretstores/azure/keyvault/go.sum @@ -50,8 +50,6 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 h1:82w8tzLcOwDP github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0/go.mod h1:S78i9yTr4o/nXlH76bKjGUye9Z2wSxO5Tz7GoDr4vfI= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 h1:Lg6BW0VPmCwcMlvOviL3ruHFO+H9tZNqscK0AeuFjGM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -59,7 +57,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -164,7 +161,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -277,7 +273,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -366,7 +361,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -577,8 +571,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -656,7 +648,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -925,7 +916,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/state/azure/blobstorage/go.mod b/tests/certification/state/azure/blobstorage/go.mod index cc146030b..6d7f6ae03 100644 --- a/tests/certification/state/azure/blobstorage/go.mod +++ b/tests/certification/state/azure/blobstorage/go.mod @@ -20,7 +20,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect diff --git a/tests/certification/state/azure/blobstorage/go.sum b/tests/certification/state/azure/blobstorage/go.sum index aceab4e19..765ba19e7 100644 --- a/tests/certification/state/azure/blobstorage/go.sum +++ b/tests/certification/state/azure/blobstorage/go.sum @@ -48,8 +48,6 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSq github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -57,7 +55,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -162,7 +159,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -275,7 +271,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -364,7 +359,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -575,8 +569,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -654,7 +646,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -923,7 +914,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/state/azure/cosmosdb/go.mod b/tests/certification/state/azure/cosmosdb/go.mod index 3b94d6670..1a78e9add 100644 --- a/tests/certification/state/azure/cosmosdb/go.mod +++ b/tests/certification/state/azure/cosmosdb/go.mod @@ -21,7 +21,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect diff --git a/tests/certification/state/azure/cosmosdb/go.sum b/tests/certification/state/azure/cosmosdb/go.sum index c0deadffe..6ef891c2c 100644 --- a/tests/certification/state/azure/cosmosdb/go.sum +++ b/tests/certification/state/azure/cosmosdb/go.sum @@ -50,8 +50,6 @@ github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3 h1:x1shk+tVZ6kLwIQMn4 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.3/go.mod h1:Fy3bbChFm4cZn6oIxYYqKB2FG3rBDxk3NZDLDJCHl+Q= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -59,7 +57,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -164,7 +161,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -277,7 +273,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -366,7 +361,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -577,8 +571,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -656,7 +648,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -925,7 +916,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/tests/certification/state/azure/tablestorage/go.mod b/tests/certification/state/azure/tablestorage/go.mod index e44425664..8007e0242 100644 --- a/tests/certification/state/azure/tablestorage/go.mod +++ b/tests/certification/state/azure/tablestorage/go.mod @@ -20,7 +20,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect - github.com/Azure/azure-storage-blob-go v0.15.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect diff --git a/tests/certification/state/azure/tablestorage/go.sum b/tests/certification/state/azure/tablestorage/go.sum index c18655f36..aa06e248b 100644 --- a/tests/certification/state/azure/tablestorage/go.sum +++ b/tests/certification/state/azure/tablestorage/go.sum @@ -48,8 +48,6 @@ github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgD github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1/go.mod h1:l3wvZkG9oW07GLBW5Cd0WwG5asOfJ8aqE8raUvNzLpk= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -57,7 +55,6 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -162,7 +159,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -275,7 +271,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -364,7 +359,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -575,8 +569,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -654,7 +646,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -923,7 +914,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= From 5ad0388de33e4166ebbfd99ce2756b3b2618c57e Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Mon, 23 Jan 2023 16:56:42 +0000 Subject: [PATCH 36/42] Fixed pubsub cert test Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../pubsub/azure/eventhubs/eventhubs_test.go | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go index 9bead326f..a4b5eb853 100644 --- a/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go +++ b/tests/certification/pubsub/azure/eventhubs/eventhubs_test.go @@ -75,6 +75,7 @@ const ( ) func TestEventhubs(t *testing.T) { + // Tests that simulate failures are un-ordered consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() consumerGroup4 := watcher.NewOrdered() @@ -90,7 +91,7 @@ func TestEventhubs(t *testing.T) { } // subscriber of the given topic - subscriberApplication := func(appID string, topicName string, messagesWatcher *watcher.Watcher) app.SetupFn { + subscriberApplication := func(appID string, topicName string, messagesWatcher *watcher.Watcher, withFailures bool) app.SetupFn { return func(ctx flow.Context, s common.Service) error { // Simulate periodic errors. sim := simulate.PeriodicError(ctx, 100) @@ -101,8 +102,10 @@ func TestEventhubs(t *testing.T) { Topic: topicName, Route: "/orders", }, func(_ context.Context, e *common.TopicEvent) (retry bool, err error) { - if err := sim(); err != nil { - return true, err + if withFailures { + if err := sim(); err != nil { + return true, err + } } // Track/Observe the data of the event. @@ -213,7 +216,7 @@ func TestEventhubs(t *testing.T) { // Test : single publisher, multiple subscriber with their own consumerID // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), - subscriberApplication(appID1, topicActiveName, consumerGroup1))). + subscriberApplication(appID1, topicActiveName, consumerGroup1, true))). // Run the Dapr sidecar with the eventhubs component 1, with permission at namespace level Step(sidecar.Run(sidecarName1, @@ -227,7 +230,7 @@ func TestEventhubs(t *testing.T) { // Run subscriberApplication app2 Step(app.Run(appID2, fmt.Sprintf(":%d", appPort+portOffset), - subscriberApplication(appID2, topicActiveName, consumerGroup2))). + subscriberApplication(appID2, topicActiveName, consumerGroup2, true))). // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, @@ -238,7 +241,7 @@ func TestEventhubs(t *testing.T) { embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset), componentRuntimeOptions(2), )). - Step("wait", flow.Sleep(10*time.Second)). + Step("wait", flow.Sleep(15*time.Second)). Step("publish messages to topic1", publishMessages(nil, sidecarName1, topicActiveName, consumerGroup1, consumerGroup2)). Step("publish messages to unUsedTopic", publishMessages(nil, sidecarName1, topicPassiveName)). Step("verify if app1 has recevied messages published to topic1", assertMessages(10*time.Second, consumerGroup1)). @@ -248,7 +251,7 @@ func TestEventhubs(t *testing.T) { // Test : multiple publisher with different partitionkey, multiple subscriber with same consumer ID // Run subscriberApplication app3 Step(app.Run(appID3, fmt.Sprintf(":%d", appPort+portOffset*2), - subscriberApplication(appID3, topicActiveName, consumerGroup2))). + subscriberApplication(appID3, topicActiveName, consumerGroup2, true))). // Run the Dapr sidecar with the component 3. Step(sidecar.Run(sidecarName3, @@ -259,7 +262,7 @@ func TestEventhubs(t *testing.T) { embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*2), componentRuntimeOptions(3), )). - Step("wait", flow.Sleep(10*time.Second)). + Step("wait", flow.Sleep(15*time.Second)). // publish message in topic1 from two publisher apps, however there are two subscriber apps (app2,app3) with same consumerID Step("publish messages to topic1 from app1", publishMessages(metadata, sidecarName1, topicActiveName, consumerGroup2)). @@ -269,7 +272,7 @@ func TestEventhubs(t *testing.T) { // Test : Entitymanagement , Test partition key, in order processing with single publisher/subscriber // Run subscriberApplication app4 Step(app.Run(appID4, fmt.Sprintf(":%d", appPort+portOffset*3), - subscriberApplication(appID4, topicToBeCreated, consumerGroup4))). + subscriberApplication(appID4, topicToBeCreated, consumerGroup4, false))). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName4, @@ -280,14 +283,14 @@ func TestEventhubs(t *testing.T) { embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*3), componentRuntimeOptions(4), )). - Step("wait", flow.Sleep(10*time.Second)). + Step("wait", flow.Sleep(15*time.Second)). Step(fmt.Sprintf("publish messages to topicToBeCreated: %s", topicToBeCreated), publishMessages(metadata, sidecarName4, topicToBeCreated, consumerGroup4)). Step("verify if app4 has recevied messages published to newly created topic", assertMessages(10*time.Second, consumerGroup4)). // Test : IOT hub // Run subscriberApplication app5 Step(app.Run(appID5, fmt.Sprintf(":%d", appPort+portOffset*4), - subscriberApplication(appID5, iotHubName, consumerGroup5))). + subscriberApplication(appID5, iotHubName, consumerGroup5, true))). // Run the Dapr sidecar with the iot component Step(sidecar.Run(sidecarName5, embedded.WithComponentsPath("./components/iotconsumer"), From a84ab2d0f6ed042a3abd679071a9877768e3715f Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Mon, 23 Jan 2023 17:39:40 +0000 Subject: [PATCH 37/42] Some fixes Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- internal/component/azure/eventhubs/metadata.go | 4 ++-- .../components/binding/consumer1/eventhubs.yaml | 2 -- tests/config/bindings/tests.yml | 2 ++ tests/conformance/bindings/bindings.go | 13 ++++++++----- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/internal/component/azure/eventhubs/metadata.go b/internal/component/azure/eventhubs/metadata.go index aa56df0b5..03dece0a1 100644 --- a/internal/component/azure/eventhubs/metadata.go +++ b/internal/component/azure/eventhubs/metadata.go @@ -78,7 +78,7 @@ func parseEventHubsMetadata(meta map[string]string, isBinding bool, log logger.L if isBinding { if m.ConnectionString == "" { if m.EventHub == "" { - return nil, errors.New("property eventHub is required when connecting with a connection string") + return nil, errors.New("property 'eventHub' is required when connecting with Azure AD") } m.hubName = m.EventHub } else { @@ -88,7 +88,7 @@ func parseEventHubsMetadata(meta map[string]string, isBinding bool, log logger.L } else if m.EventHub != "" { m.hubName = m.EventHub } else { - return nil, errors.New("the provided connection string does not contain a value for 'EntityPath' and no eventHub property was passed") + return nil, errors.New("the provided connection string does not contain a value for 'EntityPath' and no 'eventHub' property was passed") } } diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/consumer1/eventhubs.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/consumer1/eventhubs.yaml index 490d4e92b..53b3016ae 100644 --- a/tests/certification/bindings/azure/eventhubs/components/binding/consumer1/eventhubs.yaml +++ b/tests/certification/bindings/azure/eventhubs/components/binding/consumer1/eventhubs.yaml @@ -27,7 +27,5 @@ spec: secretKeyRef: name: AzureEventHubsBindingsContainer value: AzureEventHubsBindingsContainer - - name: partitionID - value: 0 auth: secretStore: envvar-secret-store \ No newline at end of file diff --git a/tests/config/bindings/tests.yml b/tests/config/bindings/tests.yml index 40791c92d..1163ddd35 100644 --- a/tests/config/bindings/tests.yml +++ b/tests/config/bindings/tests.yml @@ -29,6 +29,8 @@ components: outputData: '{"id": "$((uuid))", "orderid": "abcdef-test", "partitionKey": "partitionValue", "nestedproperty": {"subproperty": "something of value for testing"}, "description": "conformance test item"}' - component: azure.eventhubs operations: ["create", "operations", "read"] + config: + readBindingWait: 15s - component: azure.eventgrid operations: ["create", "operations", "read"] config: diff --git a/tests/conformance/bindings/bindings.go b/tests/conformance/bindings/bindings.go index 0c875d39e..f8c0e1373 100644 --- a/tests/conformance/bindings/bindings.go +++ b/tests/conformance/bindings/bindings.go @@ -19,6 +19,7 @@ import ( "io" "strconv" "strings" + "sync/atomic" "testing" "time" @@ -189,16 +190,17 @@ func ConformanceTests(t *testing.T, props map[string]string, inputBinding bindin }) } - inputBindingCall := 0 - readChan := make(chan int) + inputBindingCall := atomic.Int32{} + readChan := make(chan int, 1) readCtx, readCancel := context.WithCancel(context.Background()) defer readCancel() if config.HasOperation("read") { t.Run("read", func(t *testing.T) { testLogger.Info("Read test running ...") err := inputBinding.Read(readCtx, func(ctx context.Context, r *bindings.ReadResponse) ([]byte, error) { - inputBindingCall++ - readChan <- inputBindingCall + t.Logf("Read message: %s", string(r.Data)) + v := inputBindingCall.Add(1) + readChan <- int(v) return nil, nil }) @@ -208,6 +210,7 @@ func ConformanceTests(t *testing.T, props map[string]string, inputBinding bindin // Need a small wait here because with brokers like MQTT // if you publish before there is a consumer, the message is thrown out // Currently, there is no way to know when Read is successfully subscribed. + t.Logf("Sleeping for %v", config.ReadBindingWait) time.Sleep(config.ReadBindingWait) } @@ -262,7 +265,7 @@ func ConformanceTests(t *testing.T, props map[string]string, inputBinding bindin assert.Greater(t, inputBindingCall, 0) testLogger.Info("Read channel signalled.") case <-time.After(config.ReadBindingTimeout): - assert.Greater(t, inputBindingCall, 0) + assert.Greaterf(t, inputBindingCall, 0, "Timed out after %v while reading", config.ReadBindingTimeout) testLogger.Info("Read timeout.") } testLogger.Info("Verify Read test done.") From bd64c606297a31e3c30d81e6bbc309df965bd47f Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Mon, 23 Jan 2023 18:05:20 +0000 Subject: [PATCH 38/42] A few more tweaks Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- tests/config/bindings/tests.yml | 2 +- tests/conformance/bindings/bindings.go | 4 ++-- tests/conformance/common.go | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/config/bindings/tests.yml b/tests/config/bindings/tests.yml index 1163ddd35..d1ac1ff56 100644 --- a/tests/config/bindings/tests.yml +++ b/tests/config/bindings/tests.yml @@ -30,7 +30,7 @@ components: - component: azure.eventhubs operations: ["create", "operations", "read"] config: - readBindingWait: 15s + readBindingWait: 8s - component: azure.eventgrid operations: ["create", "operations", "read"] config: diff --git a/tests/conformance/bindings/bindings.go b/tests/conformance/bindings/bindings.go index f8c0e1373..80e131660 100644 --- a/tests/conformance/bindings/bindings.go +++ b/tests/conformance/bindings/bindings.go @@ -262,10 +262,10 @@ func ConformanceTests(t *testing.T, props map[string]string, inputBinding bindin // To stop the test from hanging if there's no response, we can setup a simple timeout. select { case <-readChan: - assert.Greater(t, inputBindingCall, 0) + assert.Greater(t, inputBindingCall.Load(), int32(0)) testLogger.Info("Read channel signalled.") case <-time.After(config.ReadBindingTimeout): - assert.Greaterf(t, inputBindingCall, 0, "Timed out after %v while reading", config.ReadBindingTimeout) + assert.Greaterf(t, inputBindingCall.Load(), int32(0), "Timed out after %v while reading", config.ReadBindingTimeout) testLogger.Info("Read timeout.") } testLogger.Info("Verify Read test done.") diff --git a/tests/conformance/common.go b/tests/conformance/common.go index 4b5b89ba5..929fdba9d 100644 --- a/tests/conformance/common.go +++ b/tests/conformance/common.go @@ -114,7 +114,12 @@ const ( ) //nolint:gochecknoglobals -var testLogger = logger.NewLogger("testLogger") +var testLogger logger.Logger + +func init() { + testLogger = logger.NewLogger("testLogger") + testLogger.SetOutputLevel(logger.DebugLevel) +} type TestConfiguration struct { ComponentType string `yaml:"componentType,omitempty"` From 899df9789999852b582f5ff2972f2eb1105f75f6 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Mon, 23 Jan 2023 18:34:50 +0000 Subject: [PATCH 39/42] Fixed conf test for binding Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- tests/config/bindings/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/config/bindings/tests.yml b/tests/config/bindings/tests.yml index d1ac1ff56..36ff0f3d0 100644 --- a/tests/config/bindings/tests.yml +++ b/tests/config/bindings/tests.yml @@ -30,7 +30,8 @@ components: - component: azure.eventhubs operations: ["create", "operations", "read"] config: - readBindingWait: 8s + # Need to wait for Event Hubs to acquire a lock, which can take up to 1m + readBindingWait: 60s - component: azure.eventgrid operations: ["create", "operations", "read"] config: From caf0c1ad057bbf06ce9c074a8429b6ff35b8f449 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Mon, 23 Jan 2023 19:22:24 +0000 Subject: [PATCH 40/42] Fixes to cert tests Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- bindings/azure/eventhubs/eventhubs.go.old | 446 ------------------ .../bindings/azure/eventhubs/README.md | 2 +- .../binding/consumer1/eventhubs.yaml | 4 +- .../binding/consumer2/eventhubs_output.yaml | 13 +- ...enthubs_partition0.yaml => eventhubs.yaml} | 8 +- .../consumer3/eventhubs_partition1.yaml | 36 -- .../components/binding/iothub/eventhubs.yaml | 4 +- .../binding/serviceprincipal/eventhubs.yaml | 17 +- .../azure/eventhubs/deleteeventhub.sh | 13 +- .../azure/eventhubs/eventhubs_test.go | 97 ++-- 10 files changed, 69 insertions(+), 571 deletions(-) delete mode 100644 bindings/azure/eventhubs/eventhubs.go.old rename tests/certification/bindings/azure/eventhubs/components/binding/consumer3/{eventhubs_partition0.yaml => eventhubs.yaml} (81%) delete mode 100644 tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs_partition1.yaml diff --git a/bindings/azure/eventhubs/eventhubs.go.old b/bindings/azure/eventhubs/eventhubs.go.old deleted file mode 100644 index 52eac099e..000000000 --- a/bindings/azure/eventhubs/eventhubs.go.old +++ /dev/null @@ -1,446 +0,0 @@ -/* -Copyright 2021 The Dapr Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package eventhubs - -import ( - "context" - "errors" - "fmt" - "strconv" - "strings" - "time" - - "github.com/Azure/azure-amqp-common-go/v4/aad" - "github.com/Azure/azure-amqp-common-go/v4/conn" - eventhub "github.com/Azure/azure-event-hubs-go/v3" - "github.com/Azure/azure-event-hubs-go/v3/eph" - "github.com/Azure/azure-event-hubs-go/v3/storage" - "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/Azure/go-autorest/autorest/azure" - - "github.com/dapr/components-contrib/bindings" - azauth "github.com/dapr/components-contrib/internal/authentication/azure" - "github.com/dapr/kit/logger" -) - -const ( - // metadata. - connectionString = "connectionString" - - // required by subscriber. - consumerGroup = "consumerGroup" - storageAccountName = "storageAccountName" - storageAccountKey = "storageAccountKey" - storageContainerName = "storageContainerName" - - // optional. - partitionKeyName = "partitionKey" - partitionIDName = "partitionID" - hubName = "eventHub" - hubNamespaceName = "eventHubNamespace" - - // errors. - hubConnectionInitErrorMsg = "error: creating eventHub hub client" - invalidConnectionStringErrorMsg = "error: connectionString is invalid" - missingHubNamespaceErrorMsg = "error: connectionString or eventHubNamespace is required" - missingHubNameErrorMsg = "error: connectionString or eventHub is required" - missingStorageAccountNameErrorMsg = "error: storageAccountName is a required attribute" - missingStorageAccountKeyErrorMsg = "error: storageAccountKey is a required attribute when connectionString is provided" - missingStorageContainerNameErrorMsg = "error: storageContainerName is a required attribute" - missingConsumerGroupErrorMsg = "error: consumerGroup is a required attribute" - - // Event Hubs SystemProperties names for metadata passthrough. - sysPropSequenceNumber = "x-opt-sequence-number" - sysPropEnqueuedTime = "x-opt-enqueued-time" - sysPropOffset = "x-opt-offset" - sysPropPartitionID = "x-opt-partition-id" - sysPropPartitionKey = "x-opt-partition-key" - sysPropIotHubDeviceConnectionID = "iothub-connection-device-id" - sysPropIotHubAuthGenerationID = "iothub-connection-auth-generation-id" - sysPropIotHubConnectionAuthMethod = "iothub-connection-auth-method" - sysPropIotHubConnectionModuleID = "iothub-connection-module-id" - sysPropIotHubEnqueuedTime = "iothub-enqueuedtime" - sysPropMessageID = "message-id" -) - -func readHandler(ctx context.Context, e *eventhub.Event, handler bindings.Handler) error { - res := bindings.ReadResponse{Data: e.Data, Metadata: map[string]string{}} - if e.SystemProperties.SequenceNumber != nil { - res.Metadata[sysPropSequenceNumber] = strconv.FormatInt(*e.SystemProperties.SequenceNumber, 10) - } - if e.SystemProperties.EnqueuedTime != nil { - res.Metadata[sysPropEnqueuedTime] = e.SystemProperties.EnqueuedTime.Format(time.RFC3339) - } - if e.SystemProperties.Offset != nil { - res.Metadata[sysPropOffset] = strconv.FormatInt(*e.SystemProperties.Offset, 10) - } - // According to azure-event-hubs-go docs, this will always be nil. - if e.SystemProperties.PartitionID != nil { - res.Metadata[sysPropPartitionID] = strconv.Itoa(int(*e.SystemProperties.PartitionID)) - } - // The following metadata properties are only present if event was generated by Azure IoT Hub. - if e.SystemProperties.PartitionKey != nil { - res.Metadata[sysPropPartitionKey] = *e.SystemProperties.PartitionKey - } - if e.SystemProperties.IoTHubDeviceConnectionID != nil { - res.Metadata[sysPropIotHubDeviceConnectionID] = *e.SystemProperties.IoTHubDeviceConnectionID - } - if e.SystemProperties.IoTHubAuthGenerationID != nil { - res.Metadata[sysPropIotHubAuthGenerationID] = *e.SystemProperties.IoTHubAuthGenerationID - } - if e.SystemProperties.IoTHubConnectionAuthMethod != nil { - res.Metadata[sysPropIotHubConnectionAuthMethod] = *e.SystemProperties.IoTHubConnectionAuthMethod - } - if e.SystemProperties.IoTHubConnectionModuleID != nil { - res.Metadata[sysPropIotHubConnectionModuleID] = *e.SystemProperties.IoTHubConnectionModuleID - } - if e.SystemProperties.IoTHubEnqueuedTime != nil { - res.Metadata[sysPropIotHubEnqueuedTime] = e.SystemProperties.IoTHubEnqueuedTime.Format(time.RFC3339) - } - // azure-event-hubs-go SDK pulls out the AMQP message-id property to the Event.ID property, map it from there. - if e.ID != "" { - res.Metadata[sysPropMessageID] = e.ID - } - _, err := handler(ctx, &res) - - return err -} - -// AzureEventHubs allows sending/receiving Azure Event Hubs events. -type AzureEventHubs struct { - hub *eventhub.Hub - metadata *azureEventHubsMetadata - eventHubSettings azauth.EnvironmentSettings - tokenProvider *aad.TokenProvider - storageCredential azblob.Credential - azureEnvironment *azure.Environment - logger logger.Logger - userAgent string -} - -type azureEventHubsMetadata struct { - connectionString string - consumerGroup string - storageAccountName string - storageAccountKey string - storageContainerName string - partitionID string - partitionKey string - eventHubName string - eventHubNamespaceName string -} - -func (m azureEventHubsMetadata) partitioned() bool { - return m.partitionID != "" -} - -// NewAzureEventHubs returns a new Azure Event hubs instance. -func NewAzureEventHubs(logger logger.Logger) bindings.InputOutputBinding { - return &AzureEventHubs{logger: logger} -} - -func validate(connectionString string) error { - _, err := conn.ParsedConnectionFromStr(connectionString) - return err -} - -func (a *AzureEventHubs) getStoragePrefixString() (string, error) { - hubName, err := a.validateAndGetHubName() - if err != nil { - return "", err - } - - // empty string in the end of slice to have a suffix "-". - return strings.Join([]string{"dapr", hubName, a.metadata.consumerGroup, ""}, "-"), nil -} - -func (a *AzureEventHubs) validateAndGetHubName() (string, error) { - hubName := a.metadata.eventHubName - if hubName == "" { - parsed, err := conn.ParsedConnectionFromStr(a.metadata.connectionString) - if err != nil { - return "", err - } - hubName = parsed.HubName - } - return hubName, nil -} - -// Init performs metadata init. -func (a *AzureEventHubs) Init(metadata bindings.Metadata) error { - m, err := parseMetadata(metadata) - if err != nil { - return err - } - - a.userAgent = "dapr-" + logger.DaprVersion - - a.metadata = m - if a.metadata.connectionString != "" { - // Validate connectionString. - validateErr := validate(a.metadata.connectionString) - if validateErr != nil { - return errors.New(invalidConnectionStringErrorMsg) - } - - var hub *eventhub.Hub - // Create partitioned sender if the partitionID is configured. - if a.metadata.partitioned() { - hub, err = eventhub.NewHubFromConnectionString(a.metadata.connectionString, - eventhub.HubWithPartitionedSender(a.metadata.partitionID), - eventhub.HubWithUserAgent(a.userAgent), - ) - } else { - hub, err = eventhub.NewHubFromConnectionString(a.metadata.connectionString, - eventhub.HubWithUserAgent(a.userAgent), - ) - } - - if err != nil { - return fmt.Errorf("unable to connect to azure event hubs: %w", err) - } - a.hub = hub - } else { - // Connect via AAD. - settings, sErr := azauth.NewEnvironmentSettings(azauth.AzureEventHubsResourceName, metadata.Properties) - if sErr != nil { - return sErr - } - a.eventHubSettings = settings - tokenProvider, tokenErr := a.eventHubSettings.GetAMQPTokenProvider() - if tokenErr != nil { - return fmt.Errorf("%s %w", hubConnectionInitErrorMsg, err) - } - a.tokenProvider = tokenProvider - - a.hub, err = eventhub.NewHub(a.metadata.eventHubNamespaceName, a.metadata.eventHubName, a.tokenProvider, eventhub.HubWithUserAgent(a.userAgent)) - if err != nil { - return fmt.Errorf("unable to connect to azure event hubs: %w", err) - } - } - - // connect to the storage account. - if m.storageAccountKey != "" { - metadata.Properties["accountKey"] = m.storageAccountKey - } - a.storageCredential, a.azureEnvironment, err = azauth.GetAzureStorageBlobCredentials(a.logger, m.storageAccountName, metadata.Properties) - if err != nil { - return fmt.Errorf("invalid credentials with error: %w", err) - } - - return nil -} - -func parseMetadata(meta bindings.Metadata) (*azureEventHubsMetadata, error) { - m := &azureEventHubsMetadata{} - - if val, ok := meta.Properties[connectionString]; ok && val != "" { - m.connectionString = val - } - - if val, ok := meta.Properties[storageAccountName]; ok && val != "" { - m.storageAccountName = val - } else { - return m, errors.New(missingStorageAccountNameErrorMsg) - } - - if val, ok := meta.Properties[storageAccountKey]; ok && val != "" { - m.storageAccountKey = val - } else if m.connectionString != "" { - return m, errors.New(missingStorageAccountKeyErrorMsg) - } - - if val, ok := meta.Properties[storageContainerName]; ok && val != "" { - m.storageContainerName = val - } else { - return m, errors.New(missingStorageContainerNameErrorMsg) - } - - if val, ok := meta.Properties[consumerGroup]; ok && val != "" { - m.consumerGroup = val - } else { - return m, errors.New(missingConsumerGroupErrorMsg) - } - - if val, ok := meta.Properties[partitionKeyName]; ok { - m.partitionKey = val - } - - if val, ok := meta.Properties[partitionIDName]; ok { - m.partitionID = val - } - - if val, ok := meta.Properties[hubName]; ok { - m.eventHubName = val - } else if m.connectionString == "" { - return m, errors.New(missingHubNameErrorMsg) - } - - if val, ok := meta.Properties[hubNamespaceName]; ok { - m.eventHubNamespaceName = val - } else if m.connectionString == "" { - return m, errors.New(missingHubNamespaceErrorMsg) - } - - return m, nil -} - -func (a *AzureEventHubs) Operations() []bindings.OperationKind { - return []bindings.OperationKind{bindings.CreateOperation} -} - -// Write posts an event hubs message. -func (a *AzureEventHubs) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*bindings.InvokeResponse, error) { - event := &eventhub.Event{ - Data: req.Data, - } - - // Send partitionKey in event. - if a.metadata.partitionKey != "" { - event.PartitionKey = &a.metadata.partitionKey - } else { - partitionKey, ok := req.Metadata[partitionKeyName] - if partitionKey != "" && ok { - event.PartitionKey = &partitionKey - } - } - - err := a.hub.Send(ctx, event) - if err != nil { - return nil, err - } - - return nil, nil -} - -// Read gets messages from eventhubs in a non-blocking way. -func (a *AzureEventHubs) Read(ctx context.Context, handler bindings.Handler) error { - if !a.metadata.partitioned() { - if err := a.RegisterEventProcessor(ctx, handler); err != nil { - return err - } - } else { - if err := a.RegisterPartitionedEventProcessor(ctx, handler); err != nil { - return err - } - } - - go func() { - // Wait for context to be canceled then close the connection - <-ctx.Done() - a.Close() - }() - - return nil -} - -// RegisterPartitionedEventProcessor - receive eventhub messages by partitionID. -func (a *AzureEventHubs) RegisterPartitionedEventProcessor(ctx context.Context, handler bindings.Handler) error { - runtimeInfo, err := a.hub.GetRuntimeInformation(ctx) - if err != nil { - return err - } - - callback := func(c context.Context, event *eventhub.Event) error { - if event == nil { - return nil - } - return readHandler(c, event, handler) - } - - ops := []eventhub.ReceiveOption{ - eventhub.ReceiveWithLatestOffset(), - } - - if a.metadata.consumerGroup != "" { - a.logger.Infof("eventhubs: using consumer group %s", a.metadata.consumerGroup) - ops = append(ops, eventhub.ReceiveWithConsumerGroup(a.metadata.consumerGroup)) - } - - if contains(runtimeInfo.PartitionIDs, a.metadata.partitionID) { - a.logger.Infof("eventhubs: using partition id %s", a.metadata.partitionID) - - _, err := a.hub.Receive(ctx, a.metadata.partitionID, callback, ops...) - if err != nil { - return err - } - } - - return nil -} - -// RegisterEventProcessor - receive eventhub messages by eventprocessor -// host by balancing partitions. -func (a *AzureEventHubs) RegisterEventProcessor(ctx context.Context, handler bindings.Handler) error { - storagePrefix, err := a.getStoragePrefixString() - if err != nil { - return err - } - - leaserPrefixOpt := storage.WithPrefixInBlobPath(storagePrefix) - leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(a.storageCredential, a.metadata.storageAccountName, a.metadata.storageContainerName, *a.azureEnvironment, leaserPrefixOpt) - if err != nil { - return err - } - - var processor *eph.EventProcessorHost - if a.metadata.connectionString != "" { - processor, err = eph.NewFromConnectionString(ctx, a.metadata.connectionString, leaserCheckpointer, leaserCheckpointer, eph.WithNoBanner(), eph.WithConsumerGroup(a.metadata.consumerGroup)) - if err != nil { - return err - } - } else { - // AAD connection. - processor, err = eph.New(ctx, a.metadata.eventHubNamespaceName, a.metadata.eventHubName, a.tokenProvider, leaserCheckpointer, leaserCheckpointer, eph.WithNoBanner(), eph.WithConsumerGroup(a.metadata.consumerGroup)) - if err != nil { - return err - } - a.logger.Debugf("processor initialized via AAD for eventHubName %s", a.metadata.eventHubName) - } - - _, err = processor.RegisterHandler( - ctx, - func(c context.Context, event *eventhub.Event) error { - return readHandler(c, event, handler) - }, - ) - if err != nil { - return err - } - - err = processor.StartNonBlocking(ctx) - if err != nil { - return err - } - - return nil -} - -func contains(arr []string, str string) bool { - for _, a := range arr { - if a == str { - return true - } - } - - return false -} - -func (a *AzureEventHubs) Close() (err error) { - // Use a background context because the connection context may be canceled already - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - err = a.hub.Close(ctx) - cancel() - return err -} diff --git a/tests/certification/bindings/azure/eventhubs/README.md b/tests/certification/bindings/azure/eventhubs/README.md index f3ceaaa6c..01e32d882 100644 --- a/tests/certification/bindings/azure/eventhubs/README.md +++ b/tests/certification/bindings/azure/eventhubs/README.md @@ -36,4 +36,4 @@ This project aims to test the Azure Event Hubs bindings component under various ### Running the tests -This must be run in the GitHub Actions Workflow configured for test infrastructure setup. \ No newline at end of file +This must be run in the GitHub Actions Workflow configured for test infrastructure setup. diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/consumer1/eventhubs.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/consumer1/eventhubs.yaml index 53b3016ae..00d926041 100644 --- a/tests/certification/bindings/azure/eventhubs/components/binding/consumer1/eventhubs.yaml +++ b/tests/certification/bindings/azure/eventhubs/components/binding/consumer1/eventhubs.yaml @@ -24,8 +24,6 @@ spec: name: AzureBlobStorageAccessKey value: AzureBlobStorageAccessKey - name: storageContainerName # Azure Storage Container Name - secretKeyRef: - name: AzureEventHubsBindingsContainer - value: AzureEventHubsBindingsContainer + value: eventhubs-bindings-container-c1 auth: secretStore: envvar-secret-store \ No newline at end of file diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/consumer2/eventhubs_output.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/consumer2/eventhubs_output.yaml index 0918d475e..91b549366 100644 --- a/tests/certification/bindings/azure/eventhubs/components/binding/consumer2/eventhubs_output.yaml +++ b/tests/certification/bindings/azure/eventhubs/components/binding/consumer2/eventhubs_output.yaml @@ -17,17 +17,6 @@ spec: secretKeyRef: name: AzureEventHubsBindingsConsumerGroup value: AzureEventHubsBindingsConsumerGroup - - name: storageAccountName # Azure Storage Account Name - secretKeyRef: - name: AzureBlobStorageAccount - value: AzureBlobStorageAccount - - name: storageAccountKey # Azure Storage Account Key - secretKeyRef: - name: AzureBlobStorageAccessKey - value: AzureBlobStorageAccessKey - - name: storageContainerName # Azure Storage Container Name - secretKeyRef: - name: AzureEventHubsBindingsContainer - value: AzureEventHubsBindingsContainer + # No storage account configuration here since this is only an output binding auth: secretStore: envvar-secret-store \ No newline at end of file diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs_partition0.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs.yaml similarity index 81% rename from tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs_partition0.yaml rename to tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs.yaml index adcb8067f..c1d6b6cf1 100644 --- a/tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs_partition0.yaml +++ b/tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs.yaml @@ -1,7 +1,7 @@ apiVersion: dapr.io/v1alpha1 kind: Component metadata: - name: azure-partition0-binding + name: azure-partitioned-binding namespace: default spec: type: bindings.azure.eventhubs @@ -26,10 +26,6 @@ spec: name: AzureBlobStorageAccessKey value: AzureBlobStorageAccessKey - name: storageContainerName # Azure Storage Container Name - secretKeyRef: - name: AzureEventHubsBindingsContainer - value: AzureEventHubsBindingsContainer - - name: partitionID - value: 0 + value: eventhubs-bindings-container-c3 auth: secretStore: envvar-secret-store \ No newline at end of file diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs_partition1.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs_partition1.yaml deleted file mode 100644 index 143f55e6e..000000000 --- a/tests/certification/bindings/azure/eventhubs/components/binding/consumer3/eventhubs_partition1.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: azure-partition1-binding - namespace: default -spec: - type: bindings.azure.eventhubs - version: v1 - metadata: - - name: consumerID - value: ehcertification2 - - name: connectionString # Azure EventHubs connection string - secretKeyRef: - name: AzureEventHubsBindingsConnectionString - value: AzureEventHubsBindingsConnectionString - - name: consumerGroup # EventHubs consumer group - secretKeyRef: - name: AzureEventHubsBindingsConsumerGroup - value: AzureEventHubsBindingsConsumerGroup - - name: storageAccountName # Azure Storage Account Name - secretKeyRef: - name: AzureBlobStorageAccount - value: AzureBlobStorageAccount - - name: storageAccountKey # Azure Storage Account Key - secretKeyRef: - name: AzureBlobStorageAccessKey - value: AzureBlobStorageAccessKey - - name: storageContainerName # Azure Storage Container Name - secretKeyRef: - name: AzureEventHubsBindingsContainer - value: AzureEventHubsBindingsContainer - - name: partitionID - value: 1 -auth: - secretStore: envvar-secret-store - \ No newline at end of file diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/iothub/eventhubs.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/iothub/eventhubs.yaml index 9cf7ac4e1..040c70f1e 100644 --- a/tests/certification/bindings/azure/eventhubs/components/binding/iothub/eventhubs.yaml +++ b/tests/certification/bindings/azure/eventhubs/components/binding/iothub/eventhubs.yaml @@ -26,8 +26,6 @@ spec: name: AzureBlobStorageAccessKey key: AzureBlobStorageAccessKey - name: storageContainerName - secretKeyRef: - name: AzureEventHubsBindingsContainer - key: AzureEventHubsBindingsContainer + value: eventhubs-bindings-container-iot auth: secretStore: envvar-secret-store diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml index bf98f0b01..9bc41c88e 100644 --- a/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml +++ b/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml @@ -7,10 +7,6 @@ spec: type: bindings.azure.eventhubs version: v1 metadata: - - name: eventHubNamespace - secretKeyRef: - name: AzureEventHubsBindingsNamespace - key: AzureEventHubsBindingsNamespace - name: connectionString secretKeyRef: name: AzureEventHubsBindingsConnectionString @@ -43,13 +39,12 @@ spec: secretKeyRef: name: AzureBlobStorageAccount key: AzureBlobStorageAccount - - name: storageAccountKey - secretKeyRef: - name: AzureBlobStorageAccessKey - key: AzureBlobStorageAccessKey + # Use Azure AD for the storage account too + #- name: storageAccountKey + # secretKeyRef: + # name: AzureBlobStorageAccessKey + # key: AzureBlobStorageAccessKey - name: storageContainerName - secretKeyRef: - name: AzureEventHubsBindingsContainer - value: AzureEventHubsBindingsContainer + value: eventhubs-bindings-container-sp auth: secretStore: envvar-secret-store diff --git a/tests/certification/bindings/azure/eventhubs/deleteeventhub.sh b/tests/certification/bindings/azure/eventhubs/deleteeventhub.sh index 9690c5bfc..550d7cb87 100755 --- a/tests/certification/bindings/azure/eventhubs/deleteeventhub.sh +++ b/tests/certification/bindings/azure/eventhubs/deleteeventhub.sh @@ -11,8 +11,19 @@ # limitations under the License. # ------------------------------------------------------------ +set -e + +CONTAINER_NAME=${1} + +if [ -z "$CONTAINER_NAME" ]; then + echo "Usage: ./deleteeventhub.sh [container-name]" + exit 1 +fi + +echo "Deleting container $CONTAINER_NAME" + # login to azure az login --service-principal -u $AzureCertificationServicePrincipalClientId -p $AzureCertificationServicePrincipalClientSecret --tenant $AzureCertificationTenantId # delete container used by the consumer -az storage container delete --account-key $AzureBlobStorageAccessKey --account-name $AzureBlobStorageAccount --name $AzureEventHubsBindingsContainer +az storage container delete --account-key $AzureBlobStorageAccessKey --account-name $AzureBlobStorageAccount --name "$CONTAINER_NAME" diff --git a/tests/certification/bindings/azure/eventhubs/eventhubs_test.go b/tests/certification/bindings/azure/eventhubs/eventhubs_test.go index a728918c0..4ffd08a17 100644 --- a/tests/certification/bindings/azure/eventhubs/eventhubs_test.go +++ b/tests/certification/bindings/azure/eventhubs/eventhubs_test.go @@ -27,16 +27,8 @@ import ( "go.uber.org/multierr" "github.com/dapr/components-contrib/bindings" - - bindings_loader "github.com/dapr/dapr/pkg/components/bindings" - "github.com/dapr/dapr/pkg/runtime" - dapr_testing "github.com/dapr/dapr/pkg/testing" - "github.com/dapr/kit/logger" - - secretstore_env "github.com/dapr/components-contrib/secretstores/local/env" - secretstores_loader "github.com/dapr/dapr/pkg/components/secretstores" - "github.com/dapr/components-contrib/bindings/azure/eventhubs" + secretstore_env "github.com/dapr/components-contrib/secretstores/local/env" "github.com/dapr/components-contrib/tests/certification/embedded" "github.com/dapr/components-contrib/tests/certification/flow" "github.com/dapr/components-contrib/tests/certification/flow/app" @@ -44,8 +36,13 @@ import ( "github.com/dapr/components-contrib/tests/certification/flow/sidecar" "github.com/dapr/components-contrib/tests/certification/flow/simulate" "github.com/dapr/components-contrib/tests/certification/flow/watcher" + bindings_loader "github.com/dapr/dapr/pkg/components/bindings" + secretstores_loader "github.com/dapr/dapr/pkg/components/secretstores" + "github.com/dapr/dapr/pkg/runtime" + dapr_testing "github.com/dapr/dapr/pkg/testing" dapr "github.com/dapr/go-sdk/client" "github.com/dapr/go-sdk/service/common" + "github.com/dapr/kit/logger" ) const ( @@ -62,7 +59,7 @@ func TestSinglePartition(t *testing.T) { httpPort := ports[1] appPort := ports[2] - consumerGroup1 := watcher.NewUnordered() + received := watcher.NewUnordered() metadata := map[string]string{ messageKey: "test", @@ -79,7 +76,7 @@ func TestSinglePartition(t *testing.T) { for i := 0; i < numMessages; i++ { outputmsg[i] = fmt.Sprintf("output binding: Message %03d", i) } - consumerGroup1.ExpectStrings(outputmsg...) + received.ExpectStrings(outputmsg...) time.Sleep(20 * time.Second) if !hasKey { metadata[messageKey] = uuid.NewString() @@ -99,7 +96,7 @@ func TestSinglePartition(t *testing.T) { } // Assert the observed messages - consumerGroup1.Assert(ctx, time.Minute) + received.Assert(ctx, time.Minute) return nil } } @@ -111,7 +108,7 @@ func TestSinglePartition(t *testing.T) { // Setup the binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("azure-single-partition-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { - consumerGroup1.Observe(string(in.Data)) + received.Observe(string(in.Data)) if err := sim(); err != nil { return nil, err } @@ -121,7 +118,7 @@ func TestSinglePartition(t *testing.T) { return err } deleteEventhub := func(ctx flow.Context) error { - output, err := exec.Command("/bin/sh", "deleteeventhub.sh").Output() + output, err := exec.Command("/bin/sh", "deleteeventhub.sh", "eventhubs-bindings-container-c1").Output() assert.Nil(t, err, "Error in deleteeventhub.sh.:\n%s", string(output)) return nil } @@ -135,9 +132,11 @@ func TestSinglePartition(t *testing.T) { embedded.WithComponentsPath("./components/binding/consumer1"), componentRuntimeOptions(), )). + Step("wait", flow.Sleep(15*time.Second)). Step("interrupt network", network.InterruptNetwork(30*time.Second, nil, nil, "443", "5671", "5672")). Step("send and wait", sendAndReceive(metadata)). Step("delete containers", deleteEventhub). + Step("wait", flow.Sleep(5*time.Second)). Run() } @@ -147,7 +146,7 @@ func TestEventhubBindingSerivcePrincipalAuth(t *testing.T) { httpPort := ports[1] appPort := ports[2] - consumerGroup1 := watcher.NewUnordered() + received := watcher.NewUnordered() metadata := map[string]string{ messageKey: "test", @@ -164,7 +163,7 @@ func TestEventhubBindingSerivcePrincipalAuth(t *testing.T) { for i := 0; i < numMessages; i++ { outputmsg[i] = fmt.Sprintf("output binding: Message %03d", i) } - consumerGroup1.ExpectStrings(outputmsg...) + received.ExpectStrings(outputmsg...) time.Sleep(20 * time.Second) if !hasKey { metadata[messageKey] = uuid.NewString() @@ -184,7 +183,7 @@ func TestEventhubBindingSerivcePrincipalAuth(t *testing.T) { } // Assert the observed messages - consumerGroup1.Assert(ctx, time.Minute) + received.Assert(ctx, time.Minute) return nil } } @@ -196,7 +195,7 @@ func TestEventhubBindingSerivcePrincipalAuth(t *testing.T) { // Setup the binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("azure-eventhubs-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { - consumerGroup1.Observe(string(in.Data)) + received.Observe(string(in.Data)) if err := sim(); err != nil { return nil, err } @@ -207,7 +206,7 @@ func TestEventhubBindingSerivcePrincipalAuth(t *testing.T) { } deleteEventhub := func(ctx flow.Context) error { - output, err := exec.Command("/bin/sh", "deleteeventhub.sh").Output() + output, err := exec.Command("/bin/sh", "deleteeventhub.sh", "eventhubs-bindings-container-sp").Output() assert.Nil(t, err, "Error in deleteeventhub.sh.:\n%s", string(output)) return nil } @@ -221,8 +220,10 @@ func TestEventhubBindingSerivcePrincipalAuth(t *testing.T) { embedded.WithComponentsPath("./components/binding/serviceprincipal"), componentRuntimeOptions(), )). + Step("wait", flow.Sleep(15*time.Second)). Step("send and wait", sendAndReceive(metadata)). Step("delete containers", deleteEventhub). + Step("wait", flow.Sleep(5*time.Second)). Run() } @@ -232,7 +233,7 @@ func TestEventhubBindingIOTHub(t *testing.T) { httpPort := ports[1] appPort := ports[2] - consumerGroup1 := watcher.NewUnordered() + received := watcher.NewUnordered() // Application logic that tracks messages from eventhub. application := func(ctx flow.Context, s common.Service) (err error) { @@ -241,7 +242,7 @@ func TestEventhubBindingIOTHub(t *testing.T) { // Setup the binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("azure-eventhubs-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { - consumerGroup1.Observe(string(in.Data)) + received.Observe(string(in.Data)) if err := sim(); err != nil { return nil, err } @@ -269,7 +270,7 @@ func TestEventhubBindingIOTHub(t *testing.T) { } } deleteEventhub := func(ctx flow.Context) error { - output, err := exec.Command("/bin/sh", "deleteeventhub.sh").Output() + output, err := exec.Command("/bin/sh", "deleteeventhub.sh", "eventhubs-bindings-container-iot").Output() assert.Nil(t, err, "Error in deleteeventhub.sh.:\n%s", string(output)) return nil } @@ -282,8 +283,10 @@ func TestEventhubBindingIOTHub(t *testing.T) { embedded.WithComponentsPath("./components/binding/iothub"), componentRuntimeOptions(), )). + Step("wait", flow.Sleep(15*time.Second)). Step("Send messages to IoT", sendIOTDevice(consumerGroup3)). Step("delete containers", deleteEventhub). + Step("wait", flow.Sleep(5*time.Second)). Run() } @@ -293,16 +296,15 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { httpPort := ports[1] appPort := ports[2] - consumerGroup1 := watcher.NewUnordered() - consumerGroup2 := watcher.NewUnordered() + received := watcher.NewUnordered() metadata0 := map[string]string{ messageKey: partition0, } - metadata1 := map[string]string{ messageKey: partition1, } + sendAndReceive := func(metadata0 map[string]string, metadata1 map[string]string) flow.Runnable { return func(ctx flow.Context) error { client, err := dapr.NewClientWithPort(fmt.Sprintf("%d", grpcPort)) @@ -313,7 +315,7 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { for i := 0; i < 50; i++ { outputmsg[i] = fmt.Sprintf("output binding: Message %d, partitionkey: %s", i, metadata0[messageKey]) } - consumerGroup1.ExpectStrings(outputmsg...) + received.ExpectStrings(outputmsg...) time.Sleep(40 * time.Second) // Send events from output binding @@ -322,7 +324,7 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { err := client.InvokeOutputBinding( ctx, &dapr.InvokeBindingRequest{ - Name: "azure-partition0-binding", + Name: "azure-partitioned-binding", Operation: "create", Data: []byte(msg), Metadata: metadata0, @@ -335,7 +337,7 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { for i := 0; i < 50; i++ { outputmsg2[i] = fmt.Sprintf("output binding: Message %d, partitionkey: %s", i+50, metadata1[messageKey]) } - consumerGroup2.ExpectStrings(outputmsg2...) + received.ExpectStrings(outputmsg2...) time.Sleep(120 * time.Second) // Send events from output binding @@ -344,7 +346,7 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { err := client.InvokeOutputBinding( ctx, &dapr.InvokeBindingRequest{ - Name: "azure-partition1-binding", + Name: "azure-partitioned-binding", Operation: "create", Data: []byte(msg2), Metadata: metadata1, @@ -353,8 +355,7 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { } // Assert the observed messages - consumerGroup1.Assert(ctx, time.Minute) - consumerGroup2.Assert(ctx, time.Minute) + received.Assert(ctx, time.Minute) return nil } } @@ -363,31 +364,20 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { // Simulate periodic errors. sim := simulate.PeriodicError(ctx, 100) // Setup the binding endpoints - err = multierr.Combine(err, - s.AddBindingInvocationHandler("azure-partition0-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { - consumerGroup1.Observe(string(in.Data)) - if err := sim(); err != nil { - return nil, err - } - consumerGroup1.FailIfNotExpected(t, string(in.Data)) - ctx.Logf("Receiving eventhubs message: %s", string(in.Data)) - return []byte("{}"), nil - }), - - s.AddBindingInvocationHandler("azure-partition1-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { - consumerGroup2.Observe(string(in.Data)) - if err := sim(); err != nil { - return nil, err - } - consumerGroup2.FailIfNotExpected(t, string(in.Data)) - ctx.Logf("Receiving eventhubs message: %s", string(in.Data)) - return []byte("{}"), nil - })) + err = s.AddBindingInvocationHandler("azure-partitioned-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + received.Observe(string(in.Data)) + if err := sim(); err != nil { + return nil, err + } + received.FailIfNotExpected(t, string(in.Data)) + ctx.Logf("Receiving eventhubs message: %s", string(in.Data)) + return []byte("{}"), nil + }) return err } deleteEventhub := func(ctx flow.Context) error { - output, err := exec.Command("/bin/sh", "deleteeventhub.sh").Output() + output, err := exec.Command("/bin/sh", "deleteeventhub.sh", "eventhubs-bindings-container-c3").Output() assert.Nil(t, err, "Error in deleteeventhub.sh.:\n%s", string(output)) return nil } @@ -402,13 +392,16 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { embedded.WithComponentsPath("./components/binding/consumer3"), componentRuntimeOptions(), )). + Step("wait", flow.Sleep(15*time.Second)). Step("send and wait", sendAndReceive(metadata0, metadata1)). Step("delete containers", deleteEventhub). + Step("wait", flow.Sleep(5*time.Second)). Run() } func componentRuntimeOptions() []runtime.Option { log := logger.NewLogger("dapr.components") + log.SetOutputLevel(logger.DebugLevel) bindingsRegistry := bindings_loader.NewRegistry() bindingsRegistry.Logger = log From f224709ec79bbbdd435346f9234db893166d0341 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Mon, 23 Jan 2023 20:09:36 +0000 Subject: [PATCH 41/42] More fixes to cert tests Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../bindings/azure/eventhubs/README.md | 2 - .../binding/serviceprincipal/eventhubs.yaml | 14 ++- .../azure/eventhubs/eventhubs_test.go | 111 ++++++++---------- 3 files changed, 60 insertions(+), 67 deletions(-) diff --git a/tests/certification/bindings/azure/eventhubs/README.md b/tests/certification/bindings/azure/eventhubs/README.md index 01e32d882..bcb2af60a 100644 --- a/tests/certification/bindings/azure/eventhubs/README.md +++ b/tests/certification/bindings/azure/eventhubs/README.md @@ -7,11 +7,9 @@ This project aims to test the Azure Event Hubs bindings component under various - Test sending /receiving data between single partition - Start an app with 1 sender and 1 receiver - Provide multiple partitions but store data in one partition - - Receiver should only receive message from one partition - Sends 100+ unique messages - Simulates periodic errors - Confirm that all expected messages were received - - Confirm that receiver does not receive messages from other than one partition - Test sending /receiving data multiple partitions/sender and receivers - Start an app with 1 sender and 1 receiver diff --git a/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml b/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml index 9bc41c88e..c74620f5f 100644 --- a/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml +++ b/tests/certification/bindings/azure/eventhubs/components/binding/serviceprincipal/eventhubs.yaml @@ -7,18 +7,22 @@ spec: type: bindings.azure.eventhubs version: v1 metadata: - - name: connectionString + - name: eventHubNamespace secretKeyRef: - name: AzureEventHubsBindingsConnectionString - value: AzureEventHubsBindingsConnectionString + name: AzureEventHubsBindingsNamespace + value: AzureEventHubsBindingsNamespace + - name: eventHub + secretKeyRef: + name: AzureEventHubsBindingsHub + value: AzureEventHubsBindingsHub - name: azureTenantId secretKeyRef: name: AzureCertificationTenantId - key: AzureCertificationTenantId + key: AzureCertificationTenantId - name: azureClientId secretKeyRef: name: AzureCertificationServicePrincipalClientId - key: AzureCertificationServicePrincipalClientId + key: AzureCertificationServicePrincipalClientId - name: azureClientSecret secretKeyRef: name: AzureCertificationServicePrincipalClientSecret diff --git a/tests/certification/bindings/azure/eventhubs/eventhubs_test.go b/tests/certification/bindings/azure/eventhubs/eventhubs_test.go index 4ffd08a17..7e801912b 100644 --- a/tests/certification/bindings/azure/eventhubs/eventhubs_test.go +++ b/tests/certification/bindings/azure/eventhubs/eventhubs_test.go @@ -18,6 +18,7 @@ import ( "fmt" "os" "os/exec" + "strconv" "testing" "time" @@ -68,7 +69,7 @@ func TestSinglePartition(t *testing.T) { sendAndReceive := func(metadata map[string]string, messages ...*watcher.Watcher) flow.Runnable { _, hasKey := metadata[messageKey] return func(ctx flow.Context) error { - client, err := dapr.NewClientWithPort(fmt.Sprintf("%d", grpcPort)) + client, err := dapr.NewClientWithPort(strconv.Itoa(grpcPort)) require.NoError(t, err, "dapr init failed") // Define what is expected @@ -108,10 +109,10 @@ func TestSinglePartition(t *testing.T) { // Setup the binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("azure-single-partition-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { - received.Observe(string(in.Data)) if err := sim(); err != nil { return nil, err } + received.Observe(string(in.Data)) ctx.Logf("Receiving eventhubs message: %s", string(in.Data)) return []byte("{}"), nil })) @@ -123,7 +124,7 @@ func TestSinglePartition(t *testing.T) { return nil } // Flow of events: Start app, sidecar, interrupt network to check reconnection, send and receive - flow.New(t, "eventhubs binding authentication using connection string single partition"). + flow.New(t, "eventhubs binding authentication using connection string"). Step(app.Run("app", fmt.Sprintf(":%d", appPort), application)). Step(sidecar.Run("sidecar", embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), @@ -135,7 +136,7 @@ func TestSinglePartition(t *testing.T) { Step("wait", flow.Sleep(15*time.Second)). Step("interrupt network", network.InterruptNetwork(30*time.Second, nil, nil, "443", "5671", "5672")). Step("send and wait", sendAndReceive(metadata)). - Step("delete containers", deleteEventhub). + Step("delete container", deleteEventhub). Step("wait", flow.Sleep(5*time.Second)). Run() } @@ -155,7 +156,7 @@ func TestEventhubBindingSerivcePrincipalAuth(t *testing.T) { sendAndReceive := func(metadata map[string]string, messages ...*watcher.Watcher) flow.Runnable { _, hasKey := metadata[messageKey] return func(ctx flow.Context) error { - client, err := dapr.NewClientWithPort(fmt.Sprintf("%d", grpcPort)) + client, err := dapr.NewClientWithPort(strconv.Itoa(grpcPort)) require.NoError(t, err, "dapr init failed") // Define what is expected @@ -195,10 +196,10 @@ func TestEventhubBindingSerivcePrincipalAuth(t *testing.T) { // Setup the binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("azure-eventhubs-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { - received.Observe(string(in.Data)) if err := sim(); err != nil { return nil, err } + received.Observe(string(in.Data)) ctx.Logf("Receiving eventhubs message: %s", string(in.Data)) return []byte("{}"), nil })) @@ -242,10 +243,10 @@ func TestEventhubBindingIOTHub(t *testing.T) { // Setup the binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("azure-eventhubs-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { - received.Observe(string(in.Data)) if err := sim(); err != nil { return nil, err } + received.Observe(string(in.Data)) ctx.Logf("Receiving eventhubs message: %s", string(in.Data)) return []byte("{}"), nil })) @@ -305,72 +306,62 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { messageKey: partition1, } - sendAndReceive := func(metadata0 map[string]string, metadata1 map[string]string) flow.Runnable { - return func(ctx flow.Context) error { - client, err := dapr.NewClientWithPort(fmt.Sprintf("%d", grpcPort)) - require.NoError(t, err, "dapr init failed") + sendAndReceive := func(ctx flow.Context) error { + client, err := dapr.NewClientWithPort(strconv.Itoa(grpcPort)) + require.NoError(t, err, "dapr init failed") - // Define what is expected - outputmsg := make([]string, 50) - for i := 0; i < 50; i++ { - outputmsg[i] = fmt.Sprintf("output binding: Message %d, partitionkey: %s", i, metadata0[messageKey]) + // Define what is expected + outputmsg := make([]string, 100) + for i := 0; i < 100; i++ { + var md map[string]string + if i < 50 { + md = metadata0 + } else { + md = metadata1 } - received.ExpectStrings(outputmsg...) - time.Sleep(40 * time.Second) - - // Send events from output binding - for _, msg := range outputmsg { - ctx.Logf("Sending eventhub message: %q", msg) - - err := client.InvokeOutputBinding( - ctx, &dapr.InvokeBindingRequest{ - Name: "azure-partitioned-binding", - Operation: "create", - Data: []byte(msg), - Metadata: metadata0, - }) - require.NoError(ctx, err, "error publishing message") - } - - // Define what is expected - outputmsg2 := make([]string, 50) - for i := 0; i < 50; i++ { - outputmsg2[i] = fmt.Sprintf("output binding: Message %d, partitionkey: %s", i+50, metadata1[messageKey]) - } - received.ExpectStrings(outputmsg2...) - time.Sleep(120 * time.Second) - - // Send events from output binding - for _, msg2 := range outputmsg2 { - ctx.Logf("Sending eventhub message: %q", msg2) - - err := client.InvokeOutputBinding( - ctx, &dapr.InvokeBindingRequest{ - Name: "azure-partitioned-binding", - Operation: "create", - Data: []byte(msg2), - Metadata: metadata1, - }) - require.NoError(ctx, err, "error publishing message") - } - - // Assert the observed messages - received.Assert(ctx, time.Minute) - return nil + outputmsg[i] = fmt.Sprintf("output binding: Message %d, partitionkey: %s", i, md[messageKey]) } + received.ExpectStrings(outputmsg...) + + // Send events from output binding + for i, msg := range outputmsg { + var md map[string]string + if i < 50 { + md = metadata0 + } else { + md = metadata1 + } + + ctx.Logf("Sending eventhub message '%s' on partition '%s'", msg, md[messageKey]) + + err := client.InvokeOutputBinding( + ctx, &dapr.InvokeBindingRequest{ + Name: "azure-partitioned-binding", + Operation: "create", + Data: []byte(msg), + Metadata: md, + }) + require.NoError(ctx, err, "error publishing message") + } + + // Assert the observed messages + time.Sleep(20 * time.Second) + received.Assert(ctx, time.Minute) + return nil } + // Application logic that tracks messages from eventhub. application := func(ctx flow.Context, s common.Service) (err error) { // Simulate periodic errors. sim := simulate.PeriodicError(ctx, 100) // Setup the binding endpoints err = s.AddBindingInvocationHandler("azure-partitioned-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { - received.Observe(string(in.Data)) if err := sim(); err != nil { return nil, err } - received.FailIfNotExpected(t, string(in.Data)) ctx.Logf("Receiving eventhubs message: %s", string(in.Data)) + received.FailIfNotExpected(t, string(in.Data)) + received.Observe(string(in.Data)) return []byte("{}"), nil }) return err @@ -393,7 +384,7 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { componentRuntimeOptions(), )). Step("wait", flow.Sleep(15*time.Second)). - Step("send and wait", sendAndReceive(metadata0, metadata1)). + Step("send and wait", sendAndReceive). Step("delete containers", deleteEventhub). Step("wait", flow.Sleep(5*time.Second)). Run() From 06b86a0fae1bd68a2b2a540823b05e14113b9e8c Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Mon, 23 Jan 2023 20:15:41 +0000 Subject: [PATCH 42/42] And more Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- tests/certification/bindings/azure/eventhubs/eventhubs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/certification/bindings/azure/eventhubs/eventhubs_test.go b/tests/certification/bindings/azure/eventhubs/eventhubs_test.go index 7e801912b..6edc6a607 100644 --- a/tests/certification/bindings/azure/eventhubs/eventhubs_test.go +++ b/tests/certification/bindings/azure/eventhubs/eventhubs_test.go @@ -324,6 +324,7 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { received.ExpectStrings(outputmsg...) // Send events from output binding + time.Sleep(20 * time.Second) for i, msg := range outputmsg { var md map[string]string if i < 50 { @@ -345,7 +346,6 @@ func TestEventhubBindingMultiplePartition(t *testing.T) { } // Assert the observed messages - time.Sleep(20 * time.Second) received.Assert(ctx, time.Minute) return nil }