Merge branch 'master' into feature/pubsub-batching

This commit is contained in:
Alessandro (Ale) Segala 2022-10-04 11:13:35 -07:00 committed by GitHub
commit 9fbf4356e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1649 additions and 1 deletions

View File

@ -15,7 +15,8 @@ for attempt in `seq $MAX_ATTEMPTS`; do
# conformance tests / contents from tests/conformance/secrets.json
if vault status &&
vault kv put secret/dapr/conftestsecret conftestsecret=abcd &&
vault kv put secret/dapr/secondsecret secondsecret=efgh;
vault kv put secret/dapr/secondsecret secondsecret=efgh &&
vault kv put secret/dapr/multiplekeyvaluessecret first=1 second=2 third=3;
then
echo ✅ secrets set;
sleep 1;

View File

@ -52,6 +52,7 @@ jobs:
- bindings.redis
- secretstores.local.env
- secretstores.local.file
- secretstores.hashicorp.vault
- bindings.rabbitmq
- bindings.localstorage
- bindings.postgres

View File

@ -0,0 +1,50 @@
# HashiCorp Vault Secret Store certification testing
This project aims to test the [HashiCorp Vault Secret Store] component under various conditions.
This secret store [supports the following features][features]:
* Basic retrieve operations
* Multiple Keys under the same secret
# Test plan
## Basic Test for CRUD operations:
1. Able to create and test connection.
2. Able to do retrieve secrets.
3. Negative test to fetch record with key, that is not present.
## Test network instability
1. Vault component does not expose a time out configuration option. For this test, let's assume a 1 minute timeout.
2. Retrieve a key to show the connection is fine.
3. Interrupt the network (Vault port, 8200) for longer than the established timeout value.
4. Wait a few seconds (less than the timeout value).
5. Try to read the key from step 2 and assert it is still there.
## Test support for multiple keys under the same secret
1. Test retrieval of secrets with multiple keys under it
## Out of scope
1. Tests verifying writing and updating a secret since secret stores do not expose this functionality.
## Running the tests
Under the current directory run:
```
go test -v vault_test.go
```
# References:
* [HashiCorp Vault Secret Store Component reference page][HashiCorp Vault Secret Store]
* [List of secret store components and their features][features]
* [PR with Conformance tests for Hashicorp Vault][conformance]
* [HashiCorp Vault API reference](https://www.vaultproject.io/api-docs)
[HashiCorp Vault Secret Store]: https://docs.dapr.io/reference/components-reference/supported-secret-stores/hashicorp-vault/
[features]: https://docs.dapr.io/reference/components-reference/supported-secret-stores/
[conformance]: https://github.com/dapr/components-contrib/pull/2031

View File

@ -0,0 +1,13 @@
apiVersion: dapr.io/v1alpha1
kind: Component
metadata:
name: my-hashicorp-vault
namespace: default
spec:
type: secretstores.hashicorp.vault
version: v1
metadata:
- name: vaultAddr
value: "http://127.0.0.1:8200"
- name: vaultToken # Matches docker compose VAULT_DEV_ROOT_TOKEN_ID env. var.
value: "vault-dev-root-token-id"

View File

@ -0,0 +1,4 @@
apiVersion: dapr.io/v1alpha1
kind: Configuration
metadata:
name: hashicorpvaultconfig

View File

@ -0,0 +1,131 @@
module github.com/dapr/components-contrib/tests/certification/secretstores/hashicorp/vault
go 1.18
require (
github.com/dapr/components-contrib v1.8.0-rc.6
github.com/dapr/components-contrib/tests/certification v0.0.0-20220526162429-d03aeba3e0d6
github.com/dapr/dapr v1.8.4-0.20220922185601-283c260751e3 // We require dapr/dapr#5208 merged
github.com/dapr/go-sdk v1.4.0
github.com/dapr/kit v0.0.2
github.com/golang/protobuf v1.5.2
github.com/stretchr/testify v1.8.0
)
require (
contrib.go.opencensus.io/exporter/prometheus v0.4.1 // indirect
github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/fasthttp/router v1.3.8 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-kit/log v0.2.0 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/google/cel-go v0.9.0 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/grandcat/zeroconf v0.0.0-20190424104450-85eadb44205c // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hashicorp/consul/api v1.11.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-hclog v1.2.1 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/serf v0.9.6 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.1 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.50 // indirect
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/openzipkin/zipkin-go v0.4.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.12.2 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.35.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/prometheus/statsd_exporter v0.22.3 // indirect
github.com/savsgio/gotils v0.0.0-20210217112953-d4a072536008 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/sony/gobreaker v0.4.2-0.20210216022020-dd874f9dd33b // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stoewer/go-strcase v1.2.0 // indirect
github.com/stretchr/objx v0.4.0 // indirect
github.com/tylertreat/comcast v1.0.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.34.0 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/otel v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.7.0 // indirect
go.opentelemetry.io/otel/sdk v1.7.0 // indirect
go.opentelemetry.io/otel/trace v1.7.0 // indirect
go.opentelemetry.io/proto/otlp v0.16.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/net v0.0.0-20220630215102-69896b714898 // indirect
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
golang.org/x/tools v0.1.11 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220622171453-ea41d75dfa0f // indirect
google.golang.org/grpc v1.48.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.23.0 // indirect
k8s.io/apiextensions-apiserver v0.23.0 // indirect
k8s.io/apimachinery v0.23.0 // indirect
k8s.io/client-go v0.23.0 // indirect
k8s.io/component-base v0.23.0 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
sigs.k8s.io/controller-runtime v0.11.0 // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
replace github.com/dapr/components-contrib/tests/certification => ../../../
replace github.com/dapr/components-contrib => ../../../../../
replace github.com/dapr/go-sdk => github.com/hunter007/dapr-go-sdk v1.3.1-0.20220709114046-2f2dc4f9a684

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,238 @@
/*
Copyright 2021 The Dapr Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package vault_test
import (
"context"
"fmt"
"testing"
"time"
"github.com/dapr/components-contrib/secretstores"
"github.com/dapr/components-contrib/secretstores/hashicorp/vault"
"github.com/dapr/components-contrib/tests/certification/embedded"
"github.com/dapr/components-contrib/tests/certification/flow"
"github.com/dapr/components-contrib/tests/certification/flow/dockercompose"
"github.com/dapr/components-contrib/tests/certification/flow/network"
"github.com/dapr/components-contrib/tests/certification/flow/sidecar"
secretstores_loader "github.com/dapr/dapr/pkg/components/secretstores"
"github.com/dapr/dapr/pkg/runtime"
dapr_testing "github.com/dapr/dapr/pkg/testing"
"github.com/dapr/go-sdk/client"
"github.com/dapr/kit/logger"
"github.com/stretchr/testify/assert"
"github.com/golang/protobuf/ptypes/empty"
)
const (
sidecarName = "hashicorp-vault-sidecar"
dockerComposeClusterYAML = "../../../../../.github/infrastructure/docker-compose-hashicorp-vault.yml"
dockerComposeProjectName = "hashicorp-vault"
secretStoreComponentPath = "./components/default"
secretStoreName = "my-hashicorp-vault" // as set in the component YAML
networkInstabilityTime = 1 * time.Minute
waitAfterInstabilityTime = networkInstabilityTime / 4
servicePortToInterrupt = "8200"
)
func TestBasicSecretRetrieval(t *testing.T) {
ports, err := dapr_testing.GetFreePorts(2)
assert.NoError(t, err)
currentGrpcPort := ports[0]
currentHttpPort := ports[1]
testGetKnownSecret := func(ctx flow.Context) error {
client, err := client.NewClientWithPort(fmt.Sprint(currentGrpcPort))
if err != nil {
panic(err)
}
defer client.Close()
emptyOpt := map[string]string{}
// This test reuses the HashiCorp Vault's conformance test resources created using
// .github/infrastructure/docker-compose-hashicorp-vault.yml,
// so it reuses the tests/conformance/secretstores/secretstores.go test secrets.
res, err := client.GetSecret(ctx, secretStoreName, "secondsecret", emptyOpt)
assert.NoError(t, err)
assert.Equal(t, "efgh", res["secondsecret"])
return nil
}
testGetMissingSecret := func(ctx flow.Context) error {
client, err := client.NewClientWithPort(fmt.Sprint(currentGrpcPort))
if err != nil {
panic(err)
}
defer client.Close()
emptyOpt := map[string]string{}
_, getErr := client.GetSecret(ctx, secretStoreName, "this_secret_is_not_there", emptyOpt)
assert.Error(t, getErr)
return nil
}
flow.New(t, "Test component is up and we can retrieve some secrets").
Step(dockercompose.Run(dockerComposeProjectName, dockerComposeClusterYAML)).
Step("Waiting for component to start...", flow.Sleep(5*time.Second)).
Step(sidecar.Run(sidecarName,
embedded.WithoutApp(),
embedded.WithComponentsPath(secretStoreComponentPath),
embedded.WithDaprGRPCPort(currentGrpcPort),
embedded.WithDaprHTTPPort(currentHttpPort),
componentRuntimeOptions(),
)).
Step("Waiting for component to load...", flow.Sleep(5*time.Second)).
Step("Verify component is registered", testComponentFound(t, secretStoreName, currentGrpcPort)).
Step("Run basic secret retrieval test", testGetKnownSecret).
Step("Test retrieval of secret that does not exist", testGetMissingSecret).
Step("Interrupt network for 1 minute",
network.InterruptNetwork(networkInstabilityTime, nil, nil, servicePortToInterrupt)).
Step("Wait for component to recover", flow.Sleep(waitAfterInstabilityTime)).
Step("Run basic test again to verify reconnection occurred", testGetKnownSecret).
Step("Stop HashiCorp Vault server", dockercompose.Stop(dockerComposeProjectName, dockerComposeClusterYAML)).
Run()
}
func TestMultipleKVRetrieval(t *testing.T) {
ports, err := dapr_testing.GetFreePorts(2)
assert.NoError(t, err)
currentGrpcPort := ports[0]
currentHttpPort := ports[1]
testGetMultipleKeyValuesFromSecret := func(ctx flow.Context) error {
client, err := client.NewClientWithPort(fmt.Sprint(currentGrpcPort))
if err != nil {
panic(err)
}
defer client.Close()
emptyOpt := map[string]string{}
// This test reuses the HashiCorp Vault's conformance test resources created using
// .github/infrastructure/docker-compose-hashicorp-vault.yml,
// so it reuses the tests/conformance/secretstores/secretstores.go test secrets.
res, err := client.GetSecret(ctx, secretStoreName, "multiplekeyvaluessecret", emptyOpt)
assert.NoError(t, err)
assert.NotNil(t, res)
assert.Equal(t, "1", res["first"])
assert.Equal(t, "2", res["second"])
assert.Equal(t, "3", res["third"])
return nil
}
flow.New(t, "Test retrieving multiple key values from a secret").
Step(dockercompose.Run(dockerComposeProjectName, dockerComposeClusterYAML)).
Step("Waiting for component to start...", flow.Sleep(5*time.Second)).
Step(sidecar.Run(sidecarName,
embedded.WithoutApp(),
embedded.WithComponentsPath(secretStoreComponentPath),
embedded.WithDaprGRPCPort(currentGrpcPort),
embedded.WithDaprHTTPPort(currentHttpPort),
componentRuntimeOptions(),
)).
Step("Waiting for component to load...", flow.Sleep(5*time.Second)).
Step("Verify component is registered", testComponentFound(t, secretStoreName, currentGrpcPort)).
Step("Verify component has support for multiple key-values under the same secret",
testComponentHasFeature(t, secretStoreName, string(secretstores.FeatureMultipleKeyValuesPerSecret), currentGrpcPort)).
Step("Test retrieval of a secret with multiple key-values", testGetMultipleKeyValuesFromSecret).
Step("Stop HashiCorp Vault server", dockercompose.Stop(dockerComposeProjectName, dockerComposeClusterYAML)).
Run()
}
func testComponentFound(t *testing.T, targetComponentName string, currentGrpcPort int) flow.Runnable {
return func(ctx flow.Context) error {
client, err := client.NewClientWithPort(fmt.Sprint(currentGrpcPort))
if err != nil {
panic(err)
}
defer client.Close()
clientCtx := context.Background()
resp, err := client.GrpcClient().GetMetadata(clientCtx, &empty.Empty{})
assert.NoError(t, err)
assert.NotNil(t, resp)
assert.NotNil(t, resp.GetRegisteredComponents())
// Find the component
componentFound := false
for _, component := range resp.GetRegisteredComponents() {
if component.GetName() == targetComponentName {
componentFound = true
break
}
}
assert.True(t, componentFound)
return nil
}
}
func testComponentHasFeature(t *testing.T, targetComponentName string, targetCapability string, currentGrpcPort int) flow.Runnable {
return func(ctx flow.Context) error {
client, err := client.NewClientWithPort(fmt.Sprint(currentGrpcPort))
if err != nil {
panic(err)
}
defer client.Close()
clientCtx := context.Background()
resp, err := client.GrpcClient().GetMetadata(clientCtx, &empty.Empty{})
assert.NoError(t, err)
assert.NotNil(t, resp)
assert.NotNil(t, resp.GetRegisteredComponents())
// Find the component
var capabilities []string = []string{}
for _, component := range resp.GetRegisteredComponents() {
if component.GetName() == targetComponentName {
capabilities = component.GetCapabilities()
break
}
}
assert.NotEmpty(t, capabilities)
// Find capability
capabilityFound := false
for _, cap := range capabilities {
if cap == targetCapability {
capabilityFound = true
break
}
}
assert.True(t, capabilityFound)
return nil
}
}
func componentRuntimeOptions() []runtime.Option {
log := logger.NewLogger("dapr.components")
secretStoreRegistry := secretstores_loader.NewRegistry()
secretStoreRegistry.Logger = log
secretStoreRegistry.RegisterComponent(vault.NewHashiCorpVaultSecretStore, "hashicorp.vault")
return []runtime.Option{
runtime.WithSecretStores(secretStoreRegistry),
}
}