Certify Azure Event Hubs as Stable in pubsub (#1623)
* First iteration for the eventhubs certifications tests Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * Fixing typos, adding more description and publishing with multiple partitions Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * Moved the secret reference in components definition to secretstore.local.env Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * moving to latest dapr/dapr github.com/dapr/dapr@v1.7.0-rc.2 Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * updating go.mod Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * changes in infra and workflow for cert eventhubs Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * typo fix in local secret store type Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * correcting the typo in policyName Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * bug fixes Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * publish to a topic with no subscriber fix Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * delete azure eventhub that got created as part of test run Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * updated readme Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * removed dummy sidecar and leveraged connectionstring at namespace level for multi topic operations Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * go mod changes and nit fixes Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * fixing iot device issue Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * NIT : CR fixes * Enabled iothub test and a bit of code refactoring Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * delete the checkpoint container for the temp eventhub Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * Revert "NIT : CR fixes" This reverts commit260f5ed686. Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * First iteration for the eventhubs certifications tests Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * Fixing typos, adding more description and publishing with multiple partitions Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * Moved the secret reference in components definition to secretstore.local.env Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * moving to latest dapr/dapr github.com/dapr/dapr@v1.7.0-rc.2 Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * updating go.mod Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * changes in infra and workflow for cert eventhubs Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * typo fix in local secret store type Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * correcting the typo in policyName Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * bug fixes Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * publish to a topic with no subscriber fix Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * delete azure eventhub that got created as part of test run Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * updated readme Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * removed dummy sidecar and leveraged connectionstring at namespace level for multi topic operations Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * go mod changes and nit fixes Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * fixing iot device issue Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * NIT : CR fixes * Enabled iothub test and a bit of code refactoring Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * delete the checkpoint container for the temp eventhub Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * Revert "NIT : CR fixes" This reverts commit260f5ed686. Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> * Fix eventhub pubsub certification test infra provisioning Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> * Fixing the scoping issue for role assignment on eh Signed-off-by: Surender Singh Malik <surenderssm@users.noreply.github.com> Co-authored-by: Bernd Verst <4535280+berndverst@users.noreply.github.com>
This commit is contained in:
parent
de97b86762
commit
37e4e8a44a
|
|
@ -15,6 +15,7 @@ param eventHubsNamespaceName string
|
|||
param rgLocation string = resourceGroup().location
|
||||
param confTestTags object = {}
|
||||
|
||||
var eventHubsNamespacePolicy = '${eventHubsNamespaceName}-namespace-policy'
|
||||
var eventHubBindingsName = '${eventHubsNamespaceName}-bindings-topic'
|
||||
var eventHubBindingsPolicyName = '${eventHubBindingsName}-policy'
|
||||
var eventHubBindingsConsumerGroupName = '${eventHubBindingsName}-cg'
|
||||
|
|
@ -23,6 +24,13 @@ var eventHubPubsubName = '${eventHubsNamespaceName}-pubsub-topic'
|
|||
var eventHubPubsubPolicyName = '${eventHubPubsubName}-policy'
|
||||
var eventHubPubsubConsumerGroupName = '${eventHubPubsubName}-cg'
|
||||
|
||||
var certificationEventHubPubsubTopicActiveName = 'certification-pubsub-topic-active'
|
||||
var certificationEventHubPubsubTopicActivePolicyName = '${certificationEventHubPubsubTopicActiveName}-policy'
|
||||
|
||||
var certificationEventHubPubsubTopicPassiveName = 'certification-pubsub-topic-passive'
|
||||
|
||||
var certificationConsumerGroupName1 = 'ehcertification1'
|
||||
var certificationConsumerGroupName2 = 'ehcertification2'
|
||||
|
||||
resource eventHubsNamespace 'Microsoft.EventHub/namespaces@2017-04-01' = {
|
||||
name: eventHubsNamespaceName
|
||||
|
|
@ -31,8 +39,21 @@ resource eventHubsNamespace 'Microsoft.EventHub/namespaces@2017-04-01' = {
|
|||
sku: {
|
||||
name: 'Standard' // For > 1 consumer group
|
||||
}
|
||||
// For connectionstring and test operation at namespace level
|
||||
resource eventHubPubsubNamespacePolicy 'authorizationRules' = {
|
||||
name: eventHubsNamespacePolicy
|
||||
properties: {
|
||||
rights: [
|
||||
'Send'
|
||||
'Listen'
|
||||
]
|
||||
}
|
||||
}
|
||||
resource eventHubBindings 'eventhubs' = {
|
||||
name: eventHubBindingsName
|
||||
properties: {
|
||||
messageRetentionInDays: 1
|
||||
}
|
||||
resource eventHubBindingsPolicy 'authorizationRules' = {
|
||||
name: eventHubBindingsPolicyName
|
||||
properties: {
|
||||
|
|
@ -48,6 +69,9 @@ resource eventHubsNamespace 'Microsoft.EventHub/namespaces@2017-04-01' = {
|
|||
}
|
||||
resource eventHubPubsub 'eventhubs' = {
|
||||
name: eventHubPubsubName
|
||||
properties: {
|
||||
messageRetentionInDays: 1
|
||||
}
|
||||
resource eventHubPubsubPolicy 'authorizationRules' = {
|
||||
name: eventHubPubsubPolicyName
|
||||
properties: {
|
||||
|
|
@ -61,6 +85,39 @@ resource eventHubsNamespace 'Microsoft.EventHub/namespaces@2017-04-01' = {
|
|||
name: eventHubPubsubConsumerGroupName
|
||||
}
|
||||
}
|
||||
resource certificationEventHubPubsubTopicActive 'eventhubs' = {
|
||||
name: certificationEventHubPubsubTopicActiveName
|
||||
properties: {
|
||||
messageRetentionInDays: 1
|
||||
}
|
||||
resource certificationEventHubPubsubTopicActivePolicy 'authorizationRules' = {
|
||||
name: certificationEventHubPubsubTopicActivePolicyName
|
||||
properties: {
|
||||
rights: [
|
||||
'Send'
|
||||
'Listen'
|
||||
]
|
||||
}
|
||||
}
|
||||
resource eventHubPubsubConsumerGroup1 'consumergroups' = {
|
||||
name: certificationConsumerGroupName1
|
||||
}
|
||||
resource eventHubPubsubConsumerGroup2 'consumergroups' = {
|
||||
name: certificationConsumerGroupName2
|
||||
}
|
||||
}
|
||||
resource certificationEventHubPubsubTopicPassive 'eventhubs' = {
|
||||
name: certificationEventHubPubsubTopicPassiveName
|
||||
properties: {
|
||||
messageRetentionInDays: 1
|
||||
}
|
||||
resource eventHubPubsubConsumerGroup1 'consumergroups' = {
|
||||
name: certificationConsumerGroupName1
|
||||
}
|
||||
resource eventHubPubsubConsumerGroup2 'consumergroups' = {
|
||||
name: certificationConsumerGroupName2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output eventHubBindingsName string = eventHubsNamespace::eventHubBindings.name
|
||||
|
|
@ -70,3 +127,7 @@ output eventHubBindingsConsumerGroupName string = eventHubsNamespace::eventHubBi
|
|||
output eventHubPubsubName string = eventHubsNamespace::eventHubPubsub.name
|
||||
output eventHubPubsubPolicyName string = eventHubsNamespace::eventHubPubsub::eventHubPubsubPolicy.name
|
||||
output eventHubPubsubConsumerGroupName string = eventHubsNamespace::eventHubPubsub::eventHubPubsubConsumerGroup.name
|
||||
|
||||
output eventHubsNamespacePolicyName string = eventHubsNamespace::eventHubPubsubNamespacePolicy.name
|
||||
output certificationEventHubPubsubTopicActiveName string = eventHubsNamespace::certificationEventHubPubsubTopicActive.name
|
||||
output certificationEventHubPubsubTopicActivePolicyName string = eventHubsNamespace::certificationEventHubPubsubTopicActive::certificationEventHubPubsubTopicActivePolicy.name
|
||||
|
|
|
|||
|
|
@ -17,14 +17,15 @@ param confTestTags object = {}
|
|||
|
||||
var iotHubBindingsConsumerGroupName = '${iotHubName}/events/bindings-cg'
|
||||
var iotHubPubsubConsumerGroupName = '${iotHubName}/events/pubsub-cg'
|
||||
var iotHubPubsubEhCertificationConsumerGroupName = '${iotHubName}/events/pubsub-eh-certification-cg'
|
||||
|
||||
resource iotHub 'Microsoft.Devices/IotHubs@2021-03-31' = {
|
||||
name: iotHubName
|
||||
location: rgLocation
|
||||
tags: confTestTags
|
||||
sku: {
|
||||
capacity: 1
|
||||
name: 'S1'
|
||||
capacity: 1
|
||||
name: 'S1'
|
||||
}
|
||||
properties: {
|
||||
eventHubEndpoints: {
|
||||
|
|
@ -56,5 +57,15 @@ resource iotHubPubsubConsumerGroup 'Microsoft.Devices/IotHubs/eventHubEndpoints/
|
|||
]
|
||||
}
|
||||
|
||||
resource iotHubPubsubEhCertificationConsumerGroup 'Microsoft.Devices/IotHubs/eventHubEndpoints/ConsumerGroups@2021-03-31' = {
|
||||
name: iotHubPubsubEhCertificationConsumerGroupName
|
||||
properties: {
|
||||
name: 'pubsub-eh-certification-cg'
|
||||
}
|
||||
dependsOn: [
|
||||
iotHub
|
||||
]
|
||||
}
|
||||
|
||||
output iotHubBindingsConsumerGroupName string = iotHubBindingsConsumerGroup.name
|
||||
output iotHubPubsubConsumerGroupName string = iotHubPubsubConsumerGroup.name
|
||||
|
|
|
|||
|
|
@ -153,6 +153,9 @@ output eventHubBindingsConsumerGroupName string = eventHubsNamespace.outputs.eve
|
|||
output eventHubPubsubName string = eventHubsNamespace.outputs.eventHubPubsubName
|
||||
output eventHubPubsubPolicyName string = eventHubsNamespace.outputs.eventHubPubsubPolicyName
|
||||
output eventHubPubsubConsumerGroupName string = eventHubsNamespace.outputs.eventHubPubsubConsumerGroupName
|
||||
output eventHubsNamespacePolicyName string = eventHubsNamespace.outputs.eventHubsNamespacePolicyName
|
||||
output certificationEventHubPubsubTopicActiveName string = eventHubsNamespace.outputs.certificationEventHubPubsubTopicActiveName
|
||||
output certificationEventHubPubsubTopicActivePolicyName string = eventHubsNamespace.outputs.certificationEventHubPubsubTopicActivePolicyName
|
||||
output iotHubName string = iotHub.name
|
||||
output iotHubBindingsConsumerGroupName string = iotHub.outputs.iotHubBindingsConsumerGroupName
|
||||
output iotHubPubsubConsumerGroupName string = iotHub.outputs.iotHubPubsubConsumerGroupName
|
||||
|
|
|
|||
|
|
@ -166,6 +166,7 @@ ACR_VAR_NAME="AzureContainerRegistryName"
|
|||
CERTIFICATION_SERVICE_PRINCIPAL_CLIENT_SECRET_VAR_NAME="AzureCertificationServicePrincipalClientSecret"
|
||||
CERTIFICATION_SERVICE_PRINCIPAL_CLIENT_ID_VAR_NAME="AzureCertificationServicePrincipalClientId"
|
||||
CERTIFICATION_TENANT_ID_VAR_NAME="AzureCertificationTenantId"
|
||||
CERTIFICATION_SUBSCRIPTION_ID_VAR_NAME="AzureCertificationSubscriptionId"
|
||||
|
||||
COSMOS_DB_VAR_NAME="AzureCosmosDB"
|
||||
COSMOS_DB_COLLECTION_VAR_NAME="AzureCosmosDBCollection"
|
||||
|
|
@ -191,6 +192,8 @@ EVENT_HUBS_PUBSUB_CONSUMER_GROUP_VAR_NAME="AzureEventHubsPubsubConsumerGroup"
|
|||
EVENT_HUBS_PUBSUB_CONTAINER_VAR_NAME="AzureEventHubsPubsubContainer"
|
||||
EVENT_HUBS_PUBSUB_NAMESPACE_VAR_NAME="AzureEventHubsPubsubNamespace"
|
||||
EVENT_HUBS_PUBSUB_HUB_VAR_NAME="AzureEventHubsPubsubHub"
|
||||
EVENT_HUBS_PUBSUB_NAMESPACE_CONNECTION_STRING_VAR_NAME="AzureEventHubsPubsubNamespaceConnectionString"
|
||||
CERTIFICATION_EVENT_HUBS_PUBSUB_TOPICACTIVE_CONNECTION_STRING_VAR_NAME="AzureEventHubsPubsubTopicActiveConnectionString"
|
||||
|
||||
IOT_HUB_NAME_VAR_NAME="AzureIotHubName"
|
||||
IOT_HUB_EVENT_HUB_CONNECTION_STRING_VAR_NAME="AzureIotHubEventHubConnectionString"
|
||||
|
|
@ -304,6 +307,18 @@ EVENT_HUB_PUBSUB_POLICY_NAME="$(az deployment sub show --name "${DEPLOY_NAME}" -
|
|||
echo "INFO: EVENT_HUB_PUBSUB_POLICY_NAME=${EVENT_HUB_PUBSUB_POLICY_NAME}"
|
||||
EVENT_HUBS_PUBSUB_CONSUMER_GROUP_NAME="$(az deployment sub show --name "${DEPLOY_NAME}" --query "properties.outputs.eventHubPubsubConsumerGroupName.value" --output tsv)"
|
||||
echo "INFO: EVENT_HUBS_PUBSUB_CONSUMER_GROUP_NAME=${EVENT_HUBS_PUBSUB_CONSUMER_GROUP_NAME}"
|
||||
|
||||
#begin : certifications pubsub.azure.eventhubs
|
||||
|
||||
EVENT_HUBS_PUB_SUB_NAMESPACE_POLICY_NAME="$(az deployment sub show --name "${DEPLOY_NAME}" --query "properties.outputs.eventHubsNamespacePolicyName.value" --output tsv)"
|
||||
echo "INFO: EVENT_HUBS_PUB_SUB_NAMESPACE_POLICY_NAME=${EVENT_HUBS_PUB_SUB_NAMESPACE_POLICY_NAME}"
|
||||
|
||||
CERTIFICATION_EVENT_HUB_PUB_SUB_TOPICACTIVE_NAME="$(az deployment sub show --name "${DEPLOY_NAME}" --query "properties.outputs.certificationEventHubPubsubTopicActiveName.value" --output tsv)"
|
||||
echo "INFO: CERTIFICATION_EVENT_HUB_PUB_SUB_TOPICACTIVE_NAME=${CERTIFICATION_EVENT_HUB_PUB_SUB_TOPICACTIVE_NAME}"
|
||||
CERTIFICATION_EVENT_HUB_PUB_SUB_TOPICACTIVE_POLICY_NAME="$(az deployment sub show --name "${DEPLOY_NAME}" --query "properties.outputs.certificationEventHubPubsubTopicActivePolicyName.value" --output tsv)"
|
||||
echo "INFO: CERTIFICATION_EVENT_HUB_PUB_SUB_TOPICACTIVE_POLICY_NAME=${CERTIFICATION_EVENT_HUB_PUB_SUB_TOPICACTIVE_POLICY_NAME}"
|
||||
#end
|
||||
|
||||
IOT_HUB_NAME="$(az deployment sub show --name "${DEPLOY_NAME}" --query "properties.outputs.iotHubName.value" --output tsv)"
|
||||
echo "INFO: IOT_HUB_NAME=${IOT_HUB_NAME}"
|
||||
IOT_HUB_BINDINGS_CONSUMER_GROUP_FULLNAME="$(az deployment sub show --name "${DEPLOY_NAME}" --query "properties.outputs.iotHubBindingsConsumerGroupName.value" --output tsv)"
|
||||
|
|
@ -612,6 +627,14 @@ EVENT_HUBS_PUBSUB_CONNECTION_STRING="$(az eventhubs eventhub authorization-rule
|
|||
echo export ${EVENT_HUBS_PUBSUB_CONNECTION_STRING_VAR_NAME}=\"${EVENT_HUBS_PUBSUB_CONNECTION_STRING}\" >> "${ENV_CONFIG_FILENAME}"
|
||||
az keyvault secret set --name "${EVENT_HUBS_PUBSUB_CONNECTION_STRING_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${EVENT_HUBS_PUBSUB_CONNECTION_STRING}"
|
||||
|
||||
EVENT_HUBS_PUBSUB_NAMESPACE_CONNECTION_STRING="$(az eventhubs namespace authorization-rule keys list --name "${EVENT_HUBS_PUB_SUB_NAMESPACE_POLICY_NAME}" --namespace-name "${EVENT_HUBS_NAMESPACE}" --resource-group "${RESOURCE_GROUP_NAME}" --query "primaryConnectionString" --output tsv)"
|
||||
echo export ${EVENT_HUBS_PUBSUB_NAMESPACE_CONNECTION_STRING_VAR_NAME}=\"${EVENT_HUBS_PUBSUB_NAMESPACE_CONNECTION_STRING}\" >> "${ENV_CONFIG_FILENAME}"
|
||||
az keyvault secret set --name "${EVENT_HUBS_PUBSUB_NAMESPACE_CONNECTION_STRING_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${EVENT_HUBS_PUBSUB_NAMESPACE_CONNECTION_STRING}"
|
||||
|
||||
CERTIFICATION_EVENT_HUBS_PUBSUB_TOPICACTIVE_CONNECTION_STRING="$(az eventhubs eventhub authorization-rule keys list --name "${CERTIFICATION_EVENT_HUB_PUB_SUB_TOPICACTIVE_POLICY_NAME}" --namespace-name "${EVENT_HUBS_NAMESPACE}" --eventhub-name "${CERTIFICATION_EVENT_HUB_PUB_SUB_TOPICACTIVE_NAME}" --resource-group "${RESOURCE_GROUP_NAME}" --query "primaryConnectionString" --output tsv)"
|
||||
echo export ${CERTIFICATION_EVENT_HUBS_PUBSUB_TOPICACTIVE_CONNECTION_STRING_VAR_NAME}=\"${CERTIFICATION_EVENT_HUBS_PUBSUB_TOPICACTIVE_CONNECTION_STRING}\" >> "${ENV_CONFIG_FILENAME}"
|
||||
az keyvault secret set --name "${CERTIFICATION_EVENT_HUBS_PUBSUB_TOPICACTIVE_CONNECTION_STRING_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${CERTIFICATION_EVENT_HUBS_PUBSUB_TOPICACTIVE_CONNECTION_STRING}"
|
||||
|
||||
echo export ${EVENT_HUBS_PUBSUB_NAMESPACE_VAR_NAME}=\"${EVENT_HUBS_NAMESPACE}\" >> "${ENV_CONFIG_FILENAME}"
|
||||
az keyvault secret set --name "${EVENT_HUBS_PUBSUB_NAMESPACE_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${EVENT_HUBS_NAMESPACE}"
|
||||
|
||||
|
|
@ -658,7 +681,10 @@ az cosmosdb sql role assignment create --account-name ${COSMOS_DB_NAME} --resour
|
|||
# Storage
|
||||
az role assignment create --assignee "${CERTIFICATION_SPAUTH_SP_PRINCIPAL_ID}" --role "Storage Blob Data Owner" --scope "/subscriptions/${SUB_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.Storage/storageAccounts/${STORAGE_NAME}"
|
||||
# Event Hubs
|
||||
az role assignment create --assignee "${CERTIFICATION_SPAUTH_SP_PRINCIPAL_ID}" --role "Azure Event Hubs Data Owner" --scope "/subscriptions/${SUB_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.EventHub/namespaces/${EVENT_HUBS_NAMESPACE}/eventhubs/*"
|
||||
az role assignment create --assignee "${CERTIFICATION_SPAUTH_SP_PRINCIPAL_ID}" --role "Azure Event Hubs Data Owner" --scope "/subscriptions/${SUB_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.EventHub/namespaces/${EVENT_HUBS_NAMESPACE}"
|
||||
# IOT hub used in eventhubs certification test
|
||||
az role assignment create --assignee "${CERTIFICATION_SPAUTH_SP_PRINCIPAL_ID}" --role "Owner" --scope "/subscriptions/${SUB_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.Devices/IotHubs/${IOT_HUB_NAME}"
|
||||
az role assignment create --assignee "${CERTIFICATION_SPAUTH_SP_PRINCIPAL_ID}" --role "IoT Hub Data Contributor" --scope "/subscriptions/${SUB_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.Devices/IotHubs/${IOT_HUB_NAME}"
|
||||
|
||||
# Now export the service principal information
|
||||
CERTIFICATION_TENANT_ID="$(az ad sp list --display-name "${CERTIFICATION_SPAUTH_SP_NAME}" --query "[].appOwnerTenantId" --output tsv)"
|
||||
|
|
@ -668,6 +694,8 @@ echo export ${CERTIFICATION_SERVICE_PRINCIPAL_CLIENT_SECRET_VAR_NAME}=\"${CERTIF
|
|||
az keyvault secret set --name "${CERTIFICATION_SERVICE_PRINCIPAL_CLIENT_SECRET_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${CERTIFICATION_SPAUTH_SP_CLIENT_SECRET}"
|
||||
echo export ${CERTIFICATION_TENANT_ID_VAR_NAME}=\"${CERTIFICATION_TENANT_ID}\" >> "${ENV_CONFIG_FILENAME}"
|
||||
az keyvault secret set --name "${CERTIFICATION_TENANT_ID_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${CERTIFICATION_TENANT_ID}"
|
||||
echo export ${CERTIFICATION_SUBSCRIPTION_ID_VAR_NAME}=\"${SUB_ID}\" >> "${ENV_CONFIG_FILENAME}"
|
||||
az keyvault secret set --name "${CERTIFICATION_SUBSCRIPTION_ID_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${SUB_ID}"
|
||||
|
||||
# ---------------------------
|
||||
# Display completion message
|
||||
|
|
|
|||
|
|
@ -93,6 +93,8 @@ jobs:
|
|||
required-secrets: AzureServiceBusConnectionString
|
||||
- component: bindings.azure.cosmosdb
|
||||
required-secrets: AzureCosmosDBUrl,AzureCosmosDB,AzureCosmosDBCollection,AzureCosmosDBMasterKey,AzureCertificationTenantId,AzureCertificationServicePrincipalClientId,AzureCertificationServicePrincipalClientSecret
|
||||
- component: pubsub.azure.eventhubs
|
||||
required-secrets: AzureEventHubsPubsubTopicActiveConnectionString,AzureEventHubsPubsubNamespace,AzureBlobStorageAccount,AzureBlobStorageAccessKey,AzureEventHubsPubsubContainer,AzureIotHubEventHubConnectionString,AzureCertificationTenantId,AzureCertificationServicePrincipalClientId,AzureCertificationServicePrincipalClientSecret,AzureResourceGroupName,AzureCertificationSubscriptionId
|
||||
EOF
|
||||
)
|
||||
echo "::set-output name=cloud-components::$CRON_COMPONENTS"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
# Azure Event Hubs Pubsub certification testing
|
||||
|
||||
This project aims to test the Azure Event Hubs pubsub component under various conditions.
|
||||
|
||||
## Test Plan
|
||||
|
||||
### Pubsub Certification Tests
|
||||
- Test partition key, in order processing single publisher/subscriber (done as part of conformance tests)
|
||||
- Test single publisher / single subscriber
|
||||
- Start an app with 1 publisher and 1 subscriber
|
||||
- The publisher publishes to 2 topics
|
||||
- The subscriber is subscribed to 1 topic
|
||||
- Test: Sends 100+ unique messages
|
||||
- App: Simulates periodic errors
|
||||
- Component: Retries on error
|
||||
- App: Observes successful messages
|
||||
- Test: Confirms that all expected messages were received
|
||||
- Test: Confirms that subscriber does not receive messages from the non-subscribed topic
|
||||
- Test single publisher / multiple subscribers with same consumerID
|
||||
- Start an app with 1 publisher and 2 subscribers
|
||||
- The publisher publishes to 1 topic with 2 partitions (EventHub scalable consumer pattern)
|
||||
- The subscriber is subscribed to 1 topic
|
||||
- Test: Sends 100+ unique messages
|
||||
- App: Simulates periodic errors
|
||||
- Component: Retries on error
|
||||
- App: Observes successful messages
|
||||
- Test: Confirms that all expected messages were received
|
||||
- Test: Confirms that each subscriber receive messages from only one partition
|
||||
- Test single publisher / multiple subscribers with different consumerIDs
|
||||
- Start an app with 1 publisher and 2 subscribers
|
||||
- The publisher publishes to 1 topic with 2 partitions (EventHub scalable consumer pattern)
|
||||
- The subscriber is subscribed to 1 topic
|
||||
- Test: Sends 100+ unique messages
|
||||
- App: Simulates periodic errors
|
||||
- Component: Retries on error
|
||||
- App: Observes successful messages
|
||||
- Test: Confirms that all expected messages were received
|
||||
- Test: Confirms that each subscriber receives messages from both partitions
|
||||
- Test multiple publishers / multiple subscribers with different consumerIDs
|
||||
- Start an app with 2 publishers and 2 subscribers
|
||||
- The publisher publishes to 1 topic with 2 partitions (EventHub scalable consumer pattern)
|
||||
- The subscriber is subscribed to 1 topic
|
||||
- Test: Sends 100+ unique messages from each publisher
|
||||
- App: Simulates periodic errors
|
||||
- Component: Retries on error
|
||||
- App: Observes successful messages
|
||||
- Test: Confirms that all expected messages were received
|
||||
- Test: Confirms that each subscriber receives messages from both partitions
|
||||
- Test entity management
|
||||
- Start a publisher and subscriber with a topic that does not exist
|
||||
- Test: Confirm creation of topic/eventHub in given eventHub namespace
|
||||
- Test: Send 100+ unique messasges to the newly created eventHub topic
|
||||
- App: Observe successful messages
|
||||
- Test: Confirm that subscriber receives all messages
|
||||
- Test IOT Event Hub : [TODO]
|
||||
- Start an app with 1 publisher and 1 subscriber
|
||||
- The publisher publishes to 1 IOT EventHub with 1 partition
|
||||
- The subscriber is subscribed to 1 topic
|
||||
- Test: Sends 100+ unique messages
|
||||
- App: Simulates periodic errors
|
||||
- Component: Retries on error
|
||||
- App: Observes successful messages
|
||||
- Test: Confirms that all expected messages were received
|
||||
- Test: Confirms that all expected system properties are set
|
||||
### Authentication Tests
|
||||
- Test connection string based authentication mechanism
|
||||
- Test AAD Service Principal based authentication
|
||||
- Utilize a service principal with appropriate roles granted
|
||||
- Test MSI based authentication : [TODO]
|
||||
- Utilize Managed Identity
|
||||
### Network Tests?
|
||||
- Simulate network interruptions [TODO : network interruptions during publish]
|
||||
- Test: Simulate network interruptions
|
||||
- Component: Begins to reconnect and resubscribe
|
||||
|
||||
### Running the tests
|
||||
|
||||
This must be run in the GitHub Actions Workflow configured for test infrastructure setup.
|
||||
|
||||
If you have access to an Azure subscription you can run this locally on Mac or Linux after running `setup-azure-conf-test.sh` in `.github/infrastructure/conformance/azure` and then sourcing the generated bash rc file.
|
||||
|
||||
One can even run the test in local by just updating metadata values related to azure resources in `components/*/eventhubs.yaml`.
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: messagebus
|
||||
namespace: default
|
||||
spec:
|
||||
type: pubsub.azure.eventhubs
|
||||
version: v1
|
||||
metadata:
|
||||
- name: consumerID
|
||||
value: ehcertification1
|
||||
- name: connectionString
|
||||
secretKeyRef:
|
||||
name: AzureEventHubsPubsubNamespaceConnectionString
|
||||
key: AzureEventHubsPubsubNamespaceConnectionString
|
||||
- name: storageAccountName
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccount
|
||||
key: AzureBlobStorageAccount
|
||||
- name: storageAccountKey
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccessKey
|
||||
key: AzureBlobStorageAccessKey
|
||||
- name: storageContainerName
|
||||
secretKeyRef:
|
||||
name: AzureEventHubsPubsubContainer
|
||||
key: AzureEventHubsPubsubContainer
|
||||
auth:
|
||||
secretstore: envvar-secret-store
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: envvar-secret-store
|
||||
namespace: default
|
||||
spec:
|
||||
type: secretstores.local.env
|
||||
version: v1
|
||||
metadata:
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: messagebus
|
||||
namespace: default
|
||||
spec:
|
||||
type: pubsub.azure.eventhubs
|
||||
version: v1
|
||||
metadata:
|
||||
- name: consumerID
|
||||
value: ehcertification2
|
||||
- name: connectionString
|
||||
secretKeyRef:
|
||||
name: AzureEventHubsPubsubTopicActiveConnectionString
|
||||
key: AzureEventHubsPubsubTopicActiveConnectionString
|
||||
- name: storageAccountName
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccount
|
||||
key: AzureBlobStorageAccount
|
||||
- name: storageAccountKey
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccessKey
|
||||
key: AzureBlobStorageAccessKey
|
||||
- name: storageContainerName
|
||||
secretKeyRef:
|
||||
name: AzureEventHubsPubsubContainer
|
||||
key: AzureEventHubsPubsubContainer
|
||||
auth:
|
||||
secretstore: envvar-secret-store
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: envvar-secret-store
|
||||
namespace: default
|
||||
spec:
|
||||
type: secretstores.local.env
|
||||
version: v1
|
||||
metadata:
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: messagebus
|
||||
namespace: default
|
||||
spec:
|
||||
type: pubsub.azure.eventhubs
|
||||
version: v1
|
||||
metadata:
|
||||
- name: consumerID
|
||||
value: ehcertification2
|
||||
- name: connectionString
|
||||
secretKeyRef:
|
||||
name: AzureEventHubsPubsubTopicActiveConnectionString
|
||||
key: AzureEventHubsPubsubTopicActiveConnectionString
|
||||
- name: storageAccountName
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccount
|
||||
key: AzureBlobStorageAccount
|
||||
- name: storageAccountKey
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccessKey
|
||||
key: AzureBlobStorageAccessKey
|
||||
- name: storageContainerName
|
||||
secretKeyRef:
|
||||
name: AzureEventHubsPubsubContainer
|
||||
key: AzureEventHubsPubsubContainer
|
||||
auth:
|
||||
secretstore: envvar-secret-store
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: envvar-secret-store
|
||||
namespace: default
|
||||
spec:
|
||||
type: secretstores.local.env
|
||||
version: v1
|
||||
metadata:
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: messagebus
|
||||
namespace: default
|
||||
spec:
|
||||
type: pubsub.azure.eventhubs
|
||||
version: v1
|
||||
metadata:
|
||||
- name: enableEntityManagement
|
||||
value: true
|
||||
- name: eventHubNamespace
|
||||
secretKeyRef:
|
||||
name: AzureEventHubsPubsubNamespace
|
||||
key: AzureEventHubsPubsubNamespace
|
||||
- name: azureTenantId
|
||||
secretKeyRef:
|
||||
name: AzureCertificationTenantId
|
||||
key: AzureCertificationTenantId
|
||||
- name: azureClientId
|
||||
secretKeyRef:
|
||||
name: AzureCertificationServicePrincipalClientId
|
||||
key: AzureCertificationServicePrincipalClientId
|
||||
- name: azureClientSecret
|
||||
secretKeyRef:
|
||||
name: AzureCertificationServicePrincipalClientSecret
|
||||
key: AzureCertificationServicePrincipalClientSecret
|
||||
- name: resourceGroupName
|
||||
secretKeyRef:
|
||||
name: AzureResourceGroupName
|
||||
key: AzureResourceGroupName
|
||||
- name: subscriptionID
|
||||
secretKeyRef:
|
||||
name: AzureCertificationSubscriptionId
|
||||
key: AzureCertificationSubscriptionId
|
||||
- name: storageAccountName
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccount
|
||||
key: AzureBlobStorageAccount
|
||||
- name: storageAccountKey
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccessKey
|
||||
key: AzureBlobStorageAccessKey
|
||||
- name: storageContainerName
|
||||
value: certificationentitymgmttest
|
||||
auth:
|
||||
secretstore: envvar-secret-store
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: envvar-secret-store
|
||||
namespace: default
|
||||
spec:
|
||||
type: secretstores.local.env
|
||||
version: v1
|
||||
metadata:
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: messagebus
|
||||
namespace: default
|
||||
spec:
|
||||
type: pubsub.azure.eventhubs
|
||||
version: v1
|
||||
metadata:
|
||||
- name: consumerID
|
||||
value: pubsub-eh-certification-cg
|
||||
- name: connectionString
|
||||
secretKeyRef:
|
||||
name: AzureIotHubEventHubConnectionString
|
||||
key: AzureIotHubEventHubConnectionString
|
||||
- name: storageAccountName
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccount
|
||||
key: AzureBlobStorageAccount
|
||||
- name: storageAccountKey
|
||||
secretKeyRef:
|
||||
name: AzureBlobStorageAccessKey
|
||||
key: AzureBlobStorageAccessKey
|
||||
- name: storageContainerName
|
||||
secretKeyRef:
|
||||
name: AzureEventHubsPubsubContainer
|
||||
key: AzureEventHubsPubsubContainer
|
||||
auth:
|
||||
secretstore: envvar-secret-store
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: envvar-secret-store
|
||||
namespace: default
|
||||
spec:
|
||||
type: secretstores.local.env
|
||||
version: v1
|
||||
metadata:
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Configuration
|
||||
metadata:
|
||||
name: pubsubroutingconfig
|
||||
spec:
|
||||
features:
|
||||
- name: PubSub.Routing
|
||||
enabled: true
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
# ------------------------------------------------------------
|
||||
# Copyright 2021 The Dapr Authors
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ------------------------------------------------------------
|
||||
|
||||
set -e
|
||||
|
||||
# This script delete the Azure Eventhub created in the scoped Azure EventHub namespace as per the test suite.
|
||||
# Step 1 : Login to Azure with provided SPN which has `Azure Event Hubs Data Owner` role on scoped eventhub namespace.
|
||||
# Step 2 : Delete the eventhub/topic which is provided as an argument
|
||||
# Step 3 : Delete the checkpoint container used by the consumer
|
||||
# how to run : ./delete-eventhub.sh topic1
|
||||
|
||||
|
||||
if [ "$#" -ne 1 ]
|
||||
then
|
||||
echo "invalid argument, eventhubname is required."
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
EVENTHUBNAME=$1
|
||||
|
||||
echo "begin: delete process for eventhub: ${EVENTHUBNAME}"
|
||||
|
||||
# login to azure
|
||||
az login --service-principal -u $AzureCertificationServicePrincipalClientId -p $AzureCertificationServicePrincipalClientSecret --tenant $AzureCertificationTenantId
|
||||
|
||||
# delete eventhub
|
||||
az eventhubs eventhub delete --resource-group $AzureResourceGroupName --namespace-name $AzureEventHubsPubsubNamespace --name $EVENTHUBNAME
|
||||
|
||||
# delete checkpoint container used by the consumer
|
||||
CONTAINERNAME="certificationentitymgmttest"
|
||||
az storage container delete --account-key $AzureBlobStorageAccessKey --account-name $AzureBlobStorageAccount --name $CONTAINERNAME
|
||||
|
||||
echo "end: delete process for eventhub: ${EVENTHUBNAME} and container ${CONTAINERNAME}"
|
||||
|
|
@ -0,0 +1,292 @@
|
|||
/*
|
||||
Copyright 2021 The Dapr Authors
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package eventhubs_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/multierr"
|
||||
|
||||
// Pub-Sub.
|
||||
"github.com/dapr/components-contrib/pubsub"
|
||||
pubsub_evethubs "github.com/dapr/components-contrib/pubsub/azure/eventhubs"
|
||||
"github.com/dapr/components-contrib/secretstores"
|
||||
secretstore_env "github.com/dapr/components-contrib/secretstores/local/env"
|
||||
pubsub_loader "github.com/dapr/dapr/pkg/components/pubsub"
|
||||
secretstores_loader "github.com/dapr/dapr/pkg/components/secretstores"
|
||||
|
||||
// Dapr runtime and Go-SDK
|
||||
"github.com/dapr/dapr/pkg/runtime"
|
||||
dapr "github.com/dapr/go-sdk/client"
|
||||
"github.com/dapr/go-sdk/service/common"
|
||||
"github.com/dapr/kit/logger"
|
||||
|
||||
// Certification testing runnables
|
||||
"github.com/dapr/components-contrib/tests/certification/embedded"
|
||||
"github.com/dapr/components-contrib/tests/certification/flow"
|
||||
"github.com/dapr/components-contrib/tests/certification/flow/app"
|
||||
"github.com/dapr/components-contrib/tests/certification/flow/sidecar"
|
||||
"github.com/dapr/components-contrib/tests/certification/flow/simulate"
|
||||
"github.com/dapr/components-contrib/tests/certification/flow/watcher"
|
||||
)
|
||||
|
||||
const (
|
||||
sidecarName1 = "dapr-1"
|
||||
sidecarName2 = "dapr-2"
|
||||
sidecarName3 = "dapr-3"
|
||||
sidecarName4 = "dapr-4"
|
||||
sidecarName5 = "dapr-5"
|
||||
|
||||
appID1 = "app-1"
|
||||
appID2 = "app-2"
|
||||
appID3 = "app-3"
|
||||
appID4 = "app-4"
|
||||
appID5 = "app-5"
|
||||
|
||||
numMessages = 100
|
||||
appPort = 8000
|
||||
portOffset = 2
|
||||
messageKey = "partitionKey"
|
||||
pubsubName = "messagebus"
|
||||
topicActiveName = "certification-pubsub-topic-active"
|
||||
topicPassiveName = "certification-pubsub-topic-passive"
|
||||
topicToBeCreated = "certification-topic-per-test-run"
|
||||
iotHubNameEnvKey = "AzureIotHubName"
|
||||
partition0 = "partition-0"
|
||||
partition1 = "partition-1"
|
||||
)
|
||||
|
||||
func TestEventhubs(t *testing.T) {
|
||||
log := logger.NewLogger("dapr.components")
|
||||
component := pubsub_loader.New("azure.eventhubs", func() pubsub.PubSub {
|
||||
return pubsub_evethubs.NewAzureEventHubs(log)
|
||||
})
|
||||
|
||||
secretStoreComponent := secretstores_loader.New("local.env", func() secretstores.SecretStore {
|
||||
return secretstore_env.NewEnvSecretStore(log)
|
||||
})
|
||||
|
||||
consumerGroup1 := watcher.NewUnordered()
|
||||
consumerGroup2 := watcher.NewUnordered()
|
||||
consumerGroup4 := watcher.NewOrdered()
|
||||
consumerGroup5 := watcher.NewUnordered()
|
||||
|
||||
// Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages.
|
||||
metadata := map[string]string{
|
||||
messageKey: partition0,
|
||||
}
|
||||
|
||||
metadata1 := map[string]string{
|
||||
messageKey: partition1,
|
||||
}
|
||||
|
||||
// subscriber of the given topic
|
||||
subscriberApplication := func(appID string, topicName string, messagesWatcher *watcher.Watcher) app.SetupFn {
|
||||
return func(ctx flow.Context, s common.Service) error {
|
||||
// Simulate periodic errors.
|
||||
sim := simulate.PeriodicError(ctx, 100)
|
||||
// Setup the /orders event handler.
|
||||
return multierr.Combine(
|
||||
s.AddTopicEventHandler(&common.Subscription{
|
||||
PubsubName: pubsubName,
|
||||
Topic: topicName,
|
||||
Route: "/orders",
|
||||
}, func(_ context.Context, e *common.TopicEvent) (retry bool, err error) {
|
||||
if err := sim(); err != nil {
|
||||
return true, err
|
||||
}
|
||||
|
||||
// Track/Observe the data of the event.
|
||||
messagesWatcher.Observe(e.Data)
|
||||
ctx.Logf("Message Received appID: %s,pubsub: %s, topic: %s, id: %s, data: %s", appID, e.PubsubName, e.Topic, e.ID, e.Data)
|
||||
return false, nil
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
publishMessages := func(metadata map[string]string, sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable {
|
||||
return func(ctx flow.Context) error {
|
||||
|
||||
// prepare the messages
|
||||
messages := make([]string, numMessages)
|
||||
for i := range messages {
|
||||
messages[i] = fmt.Sprintf("partitionKey: %s, message for topic: %s, index: %03d, uniqueId: %s", metadata[messageKey], topicName, i, uuid.New().String())
|
||||
}
|
||||
|
||||
// add the messages as expectations to the watchers
|
||||
for _, messageWatcher := range messageWatchers {
|
||||
messageWatcher.ExpectStrings(messages...)
|
||||
}
|
||||
|
||||
// get the sidecar (dapr) client
|
||||
client := sidecar.GetClient(ctx, sidecarName)
|
||||
|
||||
// publish messages
|
||||
ctx.Logf("Publishing messages. sidecarName: %s, topicName: %s", sidecarName, topicName)
|
||||
|
||||
var publishOptions dapr.PublishEventOption
|
||||
|
||||
if metadata != nil {
|
||||
publishOptions = dapr.PublishEventWithMetadata(metadata)
|
||||
}
|
||||
|
||||
for _, message := range messages {
|
||||
ctx.Logf("Publishing: %q", message)
|
||||
var err error
|
||||
|
||||
if publishOptions != nil {
|
||||
err = client.PublishEvent(ctx, pubsubName, topicName, message, publishOptions)
|
||||
} else {
|
||||
err = client.PublishEvent(ctx, pubsubName, topicName, message)
|
||||
}
|
||||
|
||||
require.NoError(ctx, err, "error publishing message")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
assertMessages := func(timeout time.Duration, messageWatchers ...*watcher.Watcher) flow.Runnable {
|
||||
return func(ctx flow.Context) error {
|
||||
// assert for messages
|
||||
for _, m := range messageWatchers {
|
||||
m.Assert(ctx, 25*timeout)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
deleteEventhub := func(ctx flow.Context) error {
|
||||
output, err := exec.Command("/bin/sh", "delete-eventhub.sh", topicToBeCreated).Output()
|
||||
assert.Nil(t, err, "Error in delete-eventhub.sh.:\n%s", string(output))
|
||||
return nil
|
||||
}
|
||||
|
||||
publishMessageAsDevice := func(messageWatchers *watcher.Watcher) flow.Runnable {
|
||||
return func(ctx flow.Context) error {
|
||||
messages := make([]string, 10)
|
||||
for i := range messages {
|
||||
messages[i] = fmt.Sprintf("testmessageForEventHubCertificationTest #%v", i+1)
|
||||
}
|
||||
messageWatchers.ExpectStrings(messages...)
|
||||
|
||||
output, err := exec.Command("/bin/sh", "send-iot-device-events.sh", topicToBeCreated).Output()
|
||||
assert.Nil(t, err, "Error in send-iot-device-events.sh.:\n%s", string(output))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// Topic name for a IOT device is same as IOTHubName
|
||||
iotHubName := os.Getenv(iotHubNameEnvKey)
|
||||
|
||||
flow.New(t, "eventhubs certification").
|
||||
|
||||
// Test : single publisher, multiple subscriber with their own consumerID
|
||||
// Run subscriberApplication app1
|
||||
Step(app.Run(appID1, fmt.Sprintf(":%d", appPort),
|
||||
subscriberApplication(appID1, topicActiveName, consumerGroup1))).
|
||||
|
||||
// Run the Dapr sidecar with the eventhubs component 1, with permission at namespace level
|
||||
Step(sidecar.Run(sidecarName1,
|
||||
embedded.WithComponentsPath("./components/consumer1"),
|
||||
embedded.WithAppProtocol(runtime.HTTPProtocol, appPort),
|
||||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort),
|
||||
runtime.WithSecretStores(secretStoreComponent),
|
||||
runtime.WithPubSubs(component))).
|
||||
|
||||
// Run subscriberApplication app2
|
||||
Step(app.Run(appID2, fmt.Sprintf(":%d", appPort+portOffset),
|
||||
subscriberApplication(appID2, topicActiveName, consumerGroup2))).
|
||||
|
||||
// Run the Dapr sidecar with the component 2.
|
||||
Step(sidecar.Run(sidecarName2,
|
||||
embedded.WithComponentsPath("./components/consumer2"),
|
||||
embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset),
|
||||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset),
|
||||
embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset),
|
||||
runtime.WithSecretStores(secretStoreComponent),
|
||||
runtime.WithPubSubs(component))).
|
||||
Step("publish messages to topic1", publishMessages(nil, sidecarName1, topicActiveName, consumerGroup1, consumerGroup2)).
|
||||
Step("publish messages to unUsedTopic", publishMessages(nil, sidecarName1, topicPassiveName)).
|
||||
Step("verify if app1 has recevied messages published to topic1", assertMessages(10*time.Second, consumerGroup1)).
|
||||
Step("verify if app2 has recevied messages published to topic1", assertMessages(10*time.Second, consumerGroup2)).
|
||||
Step("reset", flow.Reset(consumerGroup1, consumerGroup2)).
|
||||
|
||||
// Test : multiple publisher with different partitionkey, multiple subscriber with same consumer ID
|
||||
// Run subscriberApplication app3
|
||||
Step(app.Run(appID3, fmt.Sprintf(":%d", appPort+portOffset*2),
|
||||
subscriberApplication(appID3, topicActiveName, consumerGroup2))).
|
||||
|
||||
// Run the Dapr sidecar with the component 3.
|
||||
Step(sidecar.Run(sidecarName3,
|
||||
embedded.WithComponentsPath("./components/consumer3"),
|
||||
embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*2),
|
||||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*2),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*2),
|
||||
embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*2),
|
||||
runtime.WithSecretStores(secretStoreComponent),
|
||||
runtime.WithPubSubs(component))).
|
||||
|
||||
// publish message in topic1 from two publisher apps, however there are two subscriber apps (app2,app3) with same consumerID
|
||||
Step("publish messages to topic1", publishMessages(metadata, sidecarName1, topicActiveName, consumerGroup2)).
|
||||
Step("publish messages to topic1", publishMessages(metadata1, sidecarName2, topicActiveName, consumerGroup2)).
|
||||
Step("verify if app2, app3 together have recevied messages published to topic1", assertMessages(10*time.Second, consumerGroup2)).
|
||||
// Test : Entitymanagement , Test partition key, in order processing with single publisher/subscriber
|
||||
// Run subscriberApplication app4
|
||||
Step(app.Run(appID4, fmt.Sprintf(":%d", appPort+portOffset*3),
|
||||
subscriberApplication(appID4, topicToBeCreated, consumerGroup4))).
|
||||
|
||||
// Run the Dapr sidecar with the component entitymanagement
|
||||
Step(sidecar.Run(sidecarName4,
|
||||
embedded.WithComponentsPath("./components/entitymanagementconsumer"),
|
||||
embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*3),
|
||||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3),
|
||||
embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*3),
|
||||
runtime.WithSecretStores(secretStoreComponent),
|
||||
runtime.WithPubSubs(component))).
|
||||
Step(fmt.Sprintf("publish messages to topicToBeCreated: %s", topicToBeCreated), publishMessages(metadata, sidecarName4, topicToBeCreated, consumerGroup4)).
|
||||
Step("verify if app4 has recevied messages published to newly created topic", assertMessages(10*time.Second, consumerGroup4)).
|
||||
|
||||
// Test : IOT hub
|
||||
// Run subscriberApplication app5
|
||||
Step(app.Run(appID5, fmt.Sprintf(":%d", appPort+portOffset*4),
|
||||
subscriberApplication(appID5, iotHubName, consumerGroup5))).
|
||||
// Run the Dapr sidecar with the iot component
|
||||
Step(sidecar.Run(sidecarName5,
|
||||
embedded.WithComponentsPath("./components/iotconsumer"),
|
||||
embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*4),
|
||||
embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*4),
|
||||
embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*4),
|
||||
embedded.WithProfilePort(runtime.DefaultProfilePort+portOffset*4),
|
||||
runtime.WithSecretStores(secretStoreComponent),
|
||||
runtime.WithPubSubs(component))).
|
||||
Step("add expected IOT messages (simulate add message to iot)", publishMessageAsDevice(consumerGroup5)).
|
||||
Step("verify if app5 has recevied messages published to iot topic", assertMessages(40*time.Second, consumerGroup5)).
|
||||
Step("wait", flow.Sleep(5*time.Second)).
|
||||
// cleanup azure assets created as part of tests
|
||||
Step("delete eventhub created as part of the eventhub management test", deleteEventhub).
|
||||
Run()
|
||||
}
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
module github.com/dapr/components-contrib/tests/certification/pubsub/azure/eventhubs
|
||||
|
||||
go 1.17
|
||||
|
||||
require (
|
||||
github.com/dapr/components-contrib v1.7.0-rc.4
|
||||
github.com/dapr/components-contrib/tests/certification v1.4.0-rc2
|
||||
github.com/dapr/dapr v1.7.0
|
||||
github.com/dapr/go-sdk v1.4.0
|
||||
github.com/dapr/kit v0.0.2-0.20210614175626-b9074b64d233
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/stretchr/testify v1.7.0
|
||||
go.uber.org/multierr v1.7.0
|
||||
)
|
||||
|
||||
require (
|
||||
contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect
|
||||
contrib.go.opencensus.io/exporter/zipkin v0.1.1 // indirect
|
||||
github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect
|
||||
github.com/Azure/azure-amqp-common-go/v3 v3.1.0 // indirect
|
||||
github.com/Azure/azure-event-hubs-go/v3 v3.3.10 // indirect
|
||||
github.com/Azure/azure-pipeline-go v0.2.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go v59.3.0+incompatible // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 // indirect
|
||||
github.com/Azure/azure-storage-blob-go v0.10.0 // indirect
|
||||
github.com/Azure/go-amqp v0.13.1 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.11.23 // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.16 // indirect
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect
|
||||
github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
|
||||
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
|
||||
github.com/Azure/go-autorest/logger v0.2.1 // indirect
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
||||
github.com/andybalholm/brotli v1.0.2 // indirect
|
||||
github.com/antlr/antlr4 v0.0.0-20200503195918-621b933c7a7f // indirect
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.1.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/devigned/tab v0.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
|
||||
github.com/fasthttp/router v1.3.8 // indirect
|
||||
github.com/fatih/color v1.10.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.5.1 // indirect
|
||||
github.com/ghodss/yaml v1.0.0 // indirect
|
||||
github.com/go-kit/log v0.1.0 // indirect
|
||||
github.com/go-logfmt/logfmt v0.5.1 // indirect
|
||||
github.com/go-logr/logr v1.2.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/cel-go v0.9.0 // indirect
|
||||
github.com/google/go-cmp v0.5.6 // indirect
|
||||
github.com/google/gofuzz v1.1.0 // indirect
|
||||
github.com/googleapis/gnostic v0.5.5 // indirect
|
||||
github.com/gorilla/mux v1.8.0 // indirect
|
||||
github.com/grandcat/zeroconf v0.0.0-20190424104450-85eadb44205c // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/hashicorp/consul/api v1.11.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
|
||||
github.com/hashicorp/go-hclog v0.14.1 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/hashicorp/serf v0.9.5 // indirect
|
||||
github.com/imdario/mergo v0.3.12 // indirect
|
||||
github.com/jpillora/backoff v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.14.4 // indirect
|
||||
github.com/mattn/go-colorable v0.1.8 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d // indirect
|
||||
github.com/mattn/go-isatty v0.0.13 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/miekg/dns v1.1.35 // indirect
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/openzipkin/zipkin-go v0.2.2 // indirect
|
||||
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_golang v1.11.1 // indirect
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.32.1 // indirect
|
||||
github.com/prometheus/procfs v0.7.3 // indirect
|
||||
github.com/prometheus/statsd_exporter v0.22.3 // indirect
|
||||
github.com/savsgio/gotils v0.0.0-20210217112953-d4a072536008 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/sony/gobreaker v0.4.2-0.20210216022020-dd874f9dd33b // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/stoewer/go-strcase v1.2.0 // indirect
|
||||
github.com/stretchr/objx v0.3.0 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.31.1-0.20211216042702-258a4c17b4f4 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
go.opentelemetry.io/otel v0.20.0 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 // indirect
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
|
||||
golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a // indirect
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
|
||||
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
|
||||
google.golang.org/grpc v1.40.0 // indirect
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
k8s.io/api v0.23.0 // indirect
|
||||
k8s.io/apiextensions-apiserver v0.23.0 // indirect
|
||||
k8s.io/apimachinery v0.23.0 // indirect
|
||||
k8s.io/client-go v0.23.0 // indirect
|
||||
k8s.io/component-base v0.23.0 // indirect
|
||||
k8s.io/klog/v2 v2.30.0 // indirect
|
||||
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
|
||||
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
|
||||
sigs.k8s.io/controller-runtime v0.11.0 // indirect
|
||||
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
|
||||
sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
|
||||
sigs.k8s.io/yaml v1.3.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/dapr/components-contrib/tests/certification => ../../../
|
||||
|
||||
replace github.com/dapr/components-contrib => ../../../../../
|
||||
|
||||
// Uncomment for local development for testing with changes
|
||||
// in the Dapr runtime. Don't commit with this uncommented!
|
||||
//
|
||||
// replace github.com/dapr/dapr => ../../../../../dapr
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env bash
|
||||
# ------------------------------------------------------------
|
||||
# Copyright 2021 The Dapr Authors
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ------------------------------------------------------------
|
||||
|
||||
set -e
|
||||
|
||||
if [[ -z "${AzureIotHubName}" ]]; then
|
||||
echo "ERROR: AzureIotHubName environment variable not defined."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Log in to Azure using provided Service Principal (SPN) credentials
|
||||
# Required roles for SPN on IoT hub
|
||||
# `IoT Hub Data Contributor` scoped at this resource
|
||||
# `Owner` scoped at this resource
|
||||
|
||||
# login to azure
|
||||
az login --service-principal -u $AzureCertificationServicePrincipalClientId -p $AzureCertificationServicePrincipalClientSecret --tenant $AzureCertificationTenantId
|
||||
|
||||
# Create test device ID if not already present
|
||||
IOT_HUB_TEST_DEVICE_NAME="certification-test-device"
|
||||
if [[ -z "$(az iot hub device-identity show -n ${AzureIotHubName} -d ${IOT_HUB_TEST_DEVICE_NAME})" ]]; then
|
||||
az iot hub device-identity create -n ${AzureIotHubName} -d ${IOT_HUB_TEST_DEVICE_NAME}
|
||||
sleep 5
|
||||
fi
|
||||
|
||||
# Send the test IoT device messages to the IoT Hub.`testmessageForEventHubCertificationTest` is being asserted in the certification test
|
||||
# TODO : read messageCount and data as an argument
|
||||
az iot device simulate -n ${AzureIotHubName} -d ${IOT_HUB_TEST_DEVICE_NAME} --data 'testmessageForEventHubCertificationTest' --msg-count 10 --msg-interval 1 --protocol http --properties "iothub-userid=dapr-user-id;iothub-messageid=dapr-message-id"
|
||||
Loading…
Reference in New Issue