dapr
/
components-contrib
mirror of https://github.com/dapr/components-contrib.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Multiple fixes + updated docs 

Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com>
This commit is contained in:
ItalyPaleAle 2023-02-17 19:00:46 +00:00
parent fe7dd863e7
commit 2ff503a0aa
9 changed files with 44 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
.github/scripts/components-scripts/certification-pubsub.aws.dynamodb-destroy.sh → .github/scripts/components-scripts/certification-state.aws.dynamodb-destroy.sh vendored
View File

0
.github/scripts/components-scripts/certification-pubsub.aws.dynamodb-setup.sh → .github/scripts/components-scripts/certification-state.aws.dynamodb-setup.sh vendored
View File

12
.github/scripts/test-info.mjs vendored
View File

@ -159,8 +159,8 @@ const components = {
certification: true,
requireAWSCredentials: true,
requireTerraform: true,
certificationSetup: 'certification-state.aws.snssqs-setup.sh',
certificationDestroy: 'certification-state.aws.snssqs-destroy.sh',
certificationSetup: 'certification-pubsub.aws.snssqs-setup.sh',
certificationDestroy: 'certification-pubsub.aws.snssqs-destroy.sh',
},
'pubsub.aws.snssqs.docker': {
conformance: true,
@ -170,8 +170,8 @@ const components = {
conformance: true,
requireAWSCredentials: true,
requireTerraform: true,
conformanceSetup: 'conformance-state.aws.snssqs.terraform-setup.sh',
conformanceDestroy: 'conformance-state.aws.snssqs.terraform-destroy.sh',
conformanceSetup: 'conformance-pubsub.aws.snssqs.terraform-setup.sh',
conformanceDestroy: 'conformance-pubsub.aws.snssqs.terraform-destroy.sh',
},
'pubsub.azure.eventhubs': {
conformance: true,
@ -201,9 +201,9 @@ const components = {
conformance: true,
certification: true,
requiredSecrets: [
'zureServiceBusConnectionString',
'AzureServiceBusConnectionString',
'AzureServiceBusNamespace',
' AzureCertificationTenantId',
'AzureCertificationTenantId',
'AzureCertificationServicePrincipalClientId',
'AzureCertificationServicePrincipalClientSecret',
],

20
tests/certification/state/sqlite/sqlite_test.go
View File

@ -211,7 +211,7 @@ func TestSQLite(t *testing.T) {
delete(md.Properties, keyCleanupInterval)
storeObj := state_sqlite.NewSQLiteStateStore(log).(*state_sqlite.SQLiteStore)
err := storeObj.Init(md)
err := storeObj.Init(context.Background(), md)
require.NoError(t, err, "failed to init")
defer storeObj.Close()
@ -226,7 +226,7 @@ func TestSQLite(t *testing.T) {
md.Properties[keyCleanupInterval] = "10s"
storeObj := state_sqlite.NewSQLiteStateStore(log).(*state_sqlite.SQLiteStore)
err := storeObj.Init(md)
err := storeObj.Init(context.Background(), md)
require.NoError(t, err, "failed to init")
defer storeObj.Close()
@ -242,7 +242,7 @@ func TestSQLite(t *testing.T) {
md.Properties[keyCleanupInterval] = "0"
storeObj := state_sqlite.NewSQLiteStateStore(log).(*state_sqlite.SQLiteStore)
err := storeObj.Init(md)
err := storeObj.Init(context.Background(), md)
require.NoError(t, err, "failed to init")
defer storeObj.Close()
@ -272,7 +272,7 @@ func TestSQLite(t *testing.T) {
md.Properties[keyCleanupInterval] = "1s"
storeObj := state_sqlite.NewSQLiteStateStore(log).(*state_sqlite.SQLiteStore)
err := storeObj.Init(md)
err := storeObj.Init(context.Background(), md)
require.NoError(t, err, "failed to init")
defer storeObj.Close()
@ -312,7 +312,7 @@ func TestSQLite(t *testing.T) {
md.Properties[keyCleanupInterval] = "1h"
storeObj := state_sqlite.NewSQLiteStateStore(log).(*state_sqlite.SQLiteStore)
err := storeObj.Init(md)
err := storeObj.Init(context.Background(), md)
require.NoError(t, err, "failed to init")
defer storeObj.Close()
@ -386,7 +386,7 @@ func TestSQLite(t *testing.T) {
md.Properties[keyMetadataTableName] = "clean_metadata"
// Init and perform the migrations
err := storeObj.Init(md)
err := storeObj.Init(context.Background(), md)
require.NoError(t, err, "failed to init")
defer storeObj.Close()
@ -414,7 +414,7 @@ func TestSQLite(t *testing.T) {
assert.Equal(t, migrationLevel, level, "migration level mismatch: found '%s' but expected '%s'", level, migrationLevel)
// Init and perform the migrations
err = storeObj.Init(md)
err = storeObj.Init(context.Background(), md)
require.NoError(t, err, "failed to init")
defer storeObj.Close()
@ -447,7 +447,7 @@ func TestSQLite(t *testing.T) {
md.Properties[keyMetadataTableName] = "pre_metadata"
// Init and perform the migrations
err = storeObj.Init(md)
err = storeObj.Init(context.Background(), md)
require.NoError(t, err, "failed to init")
defer storeObj.Close()
@ -476,7 +476,7 @@ func TestSQLite(t *testing.T) {
// Init and perform the migrations
storeObj := state_sqlite.NewSQLiteStateStore(l).(*state_sqlite.SQLiteStore)
err := storeObj.Init(md)
err := storeObj.Init(context.Background(), md)
if err != nil {
errs <- fmt.Errorf("%d failed to init: %w", i, err)
return
@ -553,7 +553,7 @@ func TestSQLite(t *testing.T) {
var storeObjs []*state_sqlite.SQLiteStore
newStoreObj := func() error {
storeObj := state_sqlite.NewSQLiteStateStore(log).(*state_sqlite.SQLiteStore)
err := storeObj.Init(md)
err := storeObj.Init(context.Background(), md)
if err != nil {
return err
}

0
tests/config/secretstores/local/env.yml → tests/config/secretstores/local/env/env.yml vendored
View File

0
tests/config/secretstores/local/file/local.yml → tests/config/secretstores/local/file/file.yml
View File

6
tests/config/secretstores/env/tests.yml → tests/config/secretstores/tests.yml
View File

@ -1,9 +1,9 @@
# Supported operations: get, bulkget
componentType: secretstores
components:
- component: localenv
operations: ["get"]
- component: localfile
- component: local.env
allOperations: true
- component: local.file
allOperations: true
- component: azure.keyvault.certificate
allOperations: true

52
tests/conformance/README.md
View File

@ -63,17 +63,16 @@
1. Test setup is independent of the test run.
2. Run the service that needs to conformance tested locally or in your own cloud account.
- For cloud-agnostic components such as Kafka, MQTT etc., there are `docker-compose` definitions under the [/.github/infrastructure](https://github.com/dapr/components-contrib/tree/master/.github/infrastructure) folder you can use to quickly create an instance of the service. For example, to setup Kafka for conformance tests:
- For cloud-agnostic components such as Kafka, MQTT etc., there are `docker-compose` definitions under the [/.github/infrastructure](../../.github/infrastructure/) folder you can use to quickly create an instance of the service. For example, to setup Kafka for conformance tests:
```bash
docker-compose -f ./.github/infrastructure/docker-compose-kafka.yml -p kafka up -d
```
- For Azure components such as Blob Storage, Key Vault etc., there is an automation script that can help you create the resources under your subscription, and extract the environment variables needed to run the conformance tests. See [/.github/infrastructure/conformance/azure/README.md](../../.github/infrastructure/conformance/azure/README.md) for more details.
- Some components require additional set up or teardown scripts, which are placed in [/.github/scripts/components-scripts/](../../.github/scripts/components-scripts/)
> Given the variability in components and how they need to be set up for the conformance tests, you may need to refer to the [GitHub workflow for conformance tests](../../.github/workflows/conformance.yml) for any extra setup required by some components. E.g. Azure Event Grid bindings require setting up an Ngrok instance or similar endpoint for the test.
3. Some conformance tests require credentials in the form of environment variables. For examples Azure CosmosDB conformance tests will need to have Azure CosmosDB credentials. You will need to supply them to make these tests pass.
3. Some conformance tests require credentials in the form of environment variables. For examples Azure Cosmos DB conformance tests will need to have Azure Cosmos DB credentials. You will need to supply them to make these tests pass.
4. To run specific tests, run:
```bash
@ -121,22 +120,22 @@ If you want to combine VS Code & dlv for debugging so you can set breakpoints in
}
```
## Using terraform for conformance tests
## Using Terraform for conformance tests
If you are writing new conformance tests and they require cloud resources, you should use the
terraform framework we have in place. To enable your component test to use terraform there are a few changes in the normal steps you must do.
If you are writing new conformance tests and they require cloud resources, you should use the Terraform framework we have in place. To enable your component test to use terraform there are a few changes in the normal steps you must do.
1. In the `conformance.yml` you should create a new step in a workflow for your component that creates new env variables. You will need a variable for each specific resource your tests will use. If you require 3 different topics and 2 different tables for your tests you should have 5 different env variables set. The only convention you must follow for the variables is the value must use `env.UNIQUE_ID` to ensure there are no conflicts with the resource names.
1. Create a setup and teardown script in [/.github/scripts/components-scripts/](../../.github/scripts/components-scripts/) for your component. You should also define new env variables. You will need a variable for each specific resource your tests will use. If you require 3 different topics and 2 different tables for your tests you should have 5 different env variables set. The only convention you must follow for the variables is the value must use `$UNIQUE_ID` to ensure there are no conflicts with the resource names.
```bash
PUBSUB_AWS_SNSSQS_QUEUE="testQueue-${{ env.UNIQUE_ID }}"
echo "PUBSUB_AWS_SNSSQS_QUEUE=$PUBSUB_AWS_SNSSQS_QUEUE" >> $GITHUB_ENV
echo "PUBSUB_AWS_SNSSQS_QUEUE=testQueue-${UNIQUE_ID}" >> $GITHUB_ENV
```
Take a look at the AWS DynamoDB [setup](../../.github/scripts/components-scripts/conformance-state.aws.dynamodb-setup.sh) and [teardown](../../.github/scripts/components-scripts/conformance-state.aws.dynamodb-destroy.sh) scripts as example.
2. When updating the `tests.yml` defined inside `tests/config/<COMPONENT-TYPE>/` folder you should overwrite the default names of any resources the conformance tests use. These values should reference env variables which should be defined in the conformance.yml.
```yaml
- component: aws.snssqs.terraform
- component: aws.snssqs.terraform
operations: ["publish", "subscribe", "multiplehandlers"]
config:
pubsubName: aws-snssqs
@ -148,22 +147,22 @@ terraform framework we have in place. To enable your component test to use terra
3. When writing your `component.yml` you should reference your credentials using env variables and any resources specified in the yaml should use env variables as well just as you did in the `test.yml`. Also if your component has an option that controls resource creation such as `disableEntityManagement` you will need to set it so it prohibits new resource creation. We want to use only terraform to provision resources and not dapr itself for these tests.
```yaml
metadata:
metadata:
- name: accessKey
value: ${{AWS_ACCESS_KEY_ID}}
value: ${{AWS_ACCESS_KEY_ID}}
- name: secretKey
value: ${{AWS_SECRET_ACCESS_KEY}}
value: ${{AWS_SECRET_ACCESS_KEY}}
- name: region
value: "us-east-1"
value: "us-east-1"
- name: consumerID
value: ${{PUBSUB_AWS_SNSSQS_QUEUE}}
value: ${{PUBSUB_AWS_SNSSQS_QUEUE}}
- name: disableEntityManagement
value: "true"
value: "true"
```
4. You will need to create a new terrafrorm file `component.tf` to provision your resources. The file should be placed in its own folder in the `.github/infrastructure/terraform/conformance` directory such as
`.github/infrastructure/terraform/conformance/pubsub/aws/snsqsq`. The terraform file should use a UNIQUE_ID variables and use this variables when naming its resources so they matched the names defined earlier. Make sure any resources your tests will use are defined in terraform.
`.github/infrastructure/terraform/conformance/pubsub/aws/snsqsq`. The terraform file should use a `UNIQUE_ID` variable and use this variables when naming its resources so they matched the names defined earlier. Make sure any resources your tests will use are defined in terraform.
```
variable "UNIQUE_ID" {
@ -172,23 +171,18 @@ terraform framework we have in place. To enable your component test to use terra
}
```
5. The component should be added to the `cron-components` step in conformance test workflow `.github/conformance.yml`. The component should have a variable named `terraform-dir` and the value should be the relative path from `.github/infrastructure/terraform/conformance` to the folder which the tests personal terraform files are located such as `pubsub/aws/snsqsq`.
5. Register your test in the file [/.github/scripts/test-info.mjs](../../.github/scripts/test-info.mjs) file, making sure to set `requiresTerraform: true`.
```
- component: pubsub.aws.snssqs.terraform
terraform-dir: pubsub/aws/snssqs
```
## Adding new AWS component in github actions
## Adding new AWS component in GitHub Actions
1. For tests involving aws components we use a service account to provision the resources needed. If you are contributing a brand new component you will need to make sure our account has sufficient permissions to provision resources and use handle component. A Dapr STC member will have to update the service account so contact them for assistance.
2. In your component yaml for your tests you should set the component metadata properties `accesskey` and `secretkey` to the values of `${{AWS_ACCESS_KEY_ID}}` and `${{AWS_SECRET_ACCESS_KEY}}`. These env values will contain the credentials for the testing service account.
```yaml
metadata:
metadata:
- name: accessKey
value: ${{AWS_ACCESS_KEY_ID}}
value: ${{AWS_ACCESS_KEY_ID}}
- name: secretKey
value: ${{AWS_SECRET_ACCESS_KEY}}
```
value: ${{AWS_SECRET_ACCESS_KEY}}
```

5
tests/conformance/secretstores/secretstores.go
View File

@ -15,7 +15,6 @@ package secretstores
import (
"context"
"os"
"testing"
"github.com/stretchr/testify/assert"
@ -46,8 +45,8 @@ func NewTestConfig(name string, allOperations bool, operations []string) TestCon
func ConformanceTests(t *testing.T, props map[string]string, store secretstores.SecretStore, config TestConfig) {
// TODO add support for metadata
// For local env var based component test
os.Setenv("conftestsecret", "abcd")
defer os.Unsetenv("conftestsecret")
t.Setenv("conftestsecret", "abcd")
t.Setenv("secondsecret", "efgh")
// Init
t.Run("init", func(t *testing.T) {