From 4bbfb82e98fcf5a91846d3e7fca54e4457bc74b5 Mon Sep 17 00:00:00 2001 From: Anubhav Mishra Date: Sat, 2 Nov 2019 08:59:06 +0530 Subject: [PATCH] initial secretstore implementation of HashiCorp Vault (#84) * initial secretstore implementation of HashiCorp vault * updates after review * fixing golangci-lint errors * fixing temporary directory creation issues --- go.mod | 2 +- secretstores/vault/vault.go | 287 +++++++++++++++++++++++++++++++ secretstores/vault/vault_test.go | 92 ++++++++++ 3 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 secretstores/vault/vault.go create mode 100644 secretstores/vault/vault_test.go diff --git a/go.mod b/go.mod index 0313e936d..1f057e0d7 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,7 @@ require ( go.uber.org/multierr v1.2.0 // indirect golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad golang.org/x/exp v0.0.0-20190927203820-447a159532ef // indirect - golang.org/x/net v0.0.0-20190926025831-c00fd9afed17 // indirect + golang.org/x/net v0.0.0-20190926025831-c00fd9afed17 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 // indirect golang.org/x/tools v0.0.0-20191028215554-80f3f9ca0853 // indirect diff --git a/secretstores/vault/vault.go b/secretstores/vault/vault.go new file mode 100644 index 000000000..fa0252257 --- /dev/null +++ b/secretstores/vault/vault.go @@ -0,0 +1,287 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// ------------------------------------------------------------ + +package vault + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "golang.org/x/net/http2" + + "github.com/dapr/components-contrib/secretstores" +) + +const ( + defaultVaultAddress string = "https://127.0.0.1:8200" + componentVaultAddress string = "vaultAddr" + componentCaCert string = "caCert" + componentCaPath string = "caPath" + componentCaPem string = "caPem" + componentSkipVerify string = "skipVerify" + componentTLSServerName string = "tlsServerName" + componentVaultTokenMountPath string = "vaultTokenMountPath" + componentVaultKVPrefix string = "vaultKVPrefix" + defaultVaultKVPrefix string = "dapr" + vaultHTTPHeader string = "X-Vault-Token" +) + +// vaultSecretStore is a secret store implementation for HashiCorp Vault +type vaultSecretStore struct { + client *http.Client + vaultAddress string + vaultTokenMountPath string + vaultKVPrefix string +} + +// tlsConfig is TLS configuration to interact with HashiCorp Vault +type tlsConfig struct { + vaultCAPem string + vaultCACert string + vaultCAPath string + vaultSkipVerify bool + vaultServerName string +} + +// vaultKVResponse is the response data from Vault KV. +type vaultKVResponse struct { + Data struct { + Data map[string]string `json:"data"` + } `json:"data"` +} + +// NewHashiCorpVaultSecretStore returns a new HashiCorp Vault secret store +func NewHashiCorpVaultSecretStore() secretstores.SecretStore { + return &vaultSecretStore{ + client: &http.Client{}, + } +} + +// Init creates a HashiCorp Vault client +func (v *vaultSecretStore) Init(metadata secretstores.Metadata) error { + props := metadata.Properties + + // Get Vault address + address := props[componentVaultAddress] + if address == "" { + v.vaultAddress = defaultVaultAddress + } + + v.vaultAddress = address + + // Generate TLS config + tlsConf := metadataToTLSConfig(props) + + client, err := v.createHTTPClient(tlsConf) + if err != nil { + return fmt.Errorf("couldn't create client using config: %s", err) + } + + v.client = client + + tokenMountPath := props[componentVaultTokenMountPath] + if tokenMountPath == "" { + return fmt.Errorf("token mount path not set") + } + + v.vaultTokenMountPath = tokenMountPath + + vaultKVPrefix := props[componentVaultKVPrefix] + if vaultKVPrefix == "" { + vaultKVPrefix = defaultVaultKVPrefix + } + + v.vaultKVPrefix = vaultKVPrefix + + return nil +} + +func metadataToTLSConfig(props map[string]string) *tlsConfig { + tlsConf := tlsConfig{} + + // Configure TLS settings + skipVerify := props[componentSkipVerify] + tlsConf.vaultSkipVerify = false + if skipVerify == "true" { + tlsConf.vaultSkipVerify = true + } + + tlsConf.vaultCACert = props[componentCaCert] + tlsConf.vaultCAPem = props[componentCaPem] + tlsConf.vaultCAPath = props[componentCaPath] + tlsConf.vaultServerName = props[componentTLSServerName] + + return &tlsConf +} + +// GetSecret retrieves a secret using a key and returns a map of decrypted string/string values +func (v *vaultSecretStore) GetSecret(req secretstores.GetSecretRequest) (secretstores.GetSecretResponse, error) { + token, err := v.readVaultToken() + if err != nil { + return secretstores.GetSecretResponse{Data: nil}, err + } + + // Create get secret url + // TODO: Add support for versioned secrets when the secretstore request has support for it + vaultSecretPathAddr := fmt.Sprintf("%s/v1/secret/data/%s/%s?version=0", v.vaultAddress, v.vaultKVPrefix, req.Name) + + httpReq, err := http.NewRequest(http.MethodGet, vaultSecretPathAddr, nil) + // Set vault token. + httpReq.Header.Set(vaultHTTPHeader, token) + if err != nil { + return secretstores.GetSecretResponse{Data: nil}, fmt.Errorf("couldn't generate request: %s", err) + } + + httpresp, err := v.client.Do(httpReq) + if err != nil { + return secretstores.GetSecretResponse{Data: nil}, fmt.Errorf("couldn't get secret: %s", err) + } + + defer httpresp.Body.Close() + + if httpresp.StatusCode != 200 { + var b bytes.Buffer + io.Copy(&b, httpresp.Body) + return secretstores.GetSecretResponse{Data: nil}, fmt.Errorf("couldn't to get successful response: %#v, %s", + httpresp, b.String()) + } + + var d vaultKVResponse + + if err := json.NewDecoder(httpresp.Body).Decode(&d); err != nil { + return secretstores.GetSecretResponse{Data: nil}, fmt.Errorf("couldn't decode response body: %s", err) + } + + resp := secretstores.GetSecretResponse{ + Data: map[string]string{}, + } + + // Only using secret data and ignore metadata + // TODO: add support for metadata response when secretstores support it. + for k, v := range d.Data.Data { + resp.Data[k] = v + } + + return resp, nil +} + +func (v *vaultSecretStore) readVaultToken() (string, error) { + data, err := ioutil.ReadFile(v.vaultTokenMountPath) + if err != nil { + return "", fmt.Errorf("couldn't read vault token: %s", err) + } + + return string(bytes.TrimSpace(data)), nil +} + +func (v *vaultSecretStore) createHTTPClient(config *tlsConfig) (*http.Client, error) { + rootCAPools, err := v.getRootCAsPools(config.vaultCAPem, config.vaultCAPath, config.vaultCACert) + if err != nil { + return nil, err + } + + tlsClientConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: rootCAPools, + } + + if config.vaultSkipVerify { + tlsClientConfig.InsecureSkipVerify = true + } + + if config.vaultServerName != "" { + tlsClientConfig.ServerName = config.vaultServerName + } + + // Setup http transport + transport := &http.Transport{ + TLSClientConfig: tlsClientConfig, + } + + // Configure http2 client + err = http2.ConfigureTransport(transport) + if err != nil { + return nil, errors.New("failed to configure http2") + } + + return &http.Client{ + Transport: transport, + }, nil +} + +// getRootCAsPools returns root CAs when you give it CA Pem file, CA path, and CA Certificate. Default is system certificates. +func (v *vaultSecretStore) getRootCAsPools(vaultCAPem string, vaultCAPath string, vaultCACert string) (*x509.CertPool, error) { + if vaultCAPem != "" { + certPool := x509.NewCertPool() + cert := []byte(vaultCAPem) + if ok := certPool.AppendCertsFromPEM(cert); !ok { + return nil, fmt.Errorf("couldn't read PEM") + } + return certPool, nil + } + + if vaultCAPath != "" { + certPool := x509.NewCertPool() + if err := readCertificateFolder(certPool, vaultCAPath); err != nil { + return nil, err + } + return certPool, nil + } + + if vaultCACert != "" { + certPool := x509.NewCertPool() + if err := readCertificateFile(certPool, vaultCACert); err != nil { + return nil, err + } + return certPool, nil + } + + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("couldn't read system certs: %s", err) + } + + return certPool, err +} + +// readCertificateFile reads the certificate at given path +func readCertificateFile(certPool *x509.CertPool, path string) error { + // Read certificate file + pemFile, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("couldn't read CA file from disk: %s", err) + } + + if ok := certPool.AppendCertsFromPEM(pemFile); !ok { + return fmt.Errorf("couldn't read PEM") + } + + return nil +} + +// readCertificateFolder scans a folder for certificates +func readCertificateFolder(certPool *x509.CertPool, path string) error { + err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + + return readCertificateFile(certPool, p) + }) + + if err != nil { + return fmt.Errorf("couldn't read certificates at %s: %s", path, err) + } + return nil +} diff --git a/secretstores/vault/vault_test.go b/secretstores/vault/vault_test.go new file mode 100644 index 000000000..0ba3ad697 --- /dev/null +++ b/secretstores/vault/vault_test.go @@ -0,0 +1,92 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// ------------------------------------------------------------ + +package vault + +import ( + "encoding/base64" + "io/ioutil" + "os" + "strconv" + "testing" + + "github.com/dapr/components-contrib/secretstores" + "github.com/stretchr/testify/assert" +) + +const ( + + // base64 encoded certificate + certificate = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURVakNDQWpvQ0NRRFlZdzdMeXN4VXRUQU5CZ2txaGtpRzl3MEJBUXNGQURCck1Rc3dDUVlEVlFRR0V3SkQKUVRFWk1CY0dBMVVFQ0F3UVFuSnBkR2x6YUNCRGIyeDFiV0pwWVRFU01CQUdBMVVFQnd3SlZtRnVZMjkxZG1WeQpNUk13RVFZRFZRUUtEQXB0YVhOb2NtRmpiM0p3TVJnd0ZnWURWUVFEREE5MllYVnNkSEJ5YjJwbFkzUXVhVzh3CkhoY05NVGt4TVRBeE1UQTBPREV5V2hjTk1qQXhNRE14TVRBME9ERXlXakJyTVFzd0NRWURWUVFHRXdKRFFURVoKTUJjR0ExVUVDQXdRUW5KcGRHbHphQ0JEYjJ4MWJXSnBZVEVTTUJBR0ExVUVCd3dKVm1GdVkyOTFkbVZ5TVJNdwpFUVlEVlFRS0RBcHRhWE5vY21GamIzSndNUmd3RmdZRFZRUUREQTkyWVhWc2RIQnliMnBsWTNRdWFXOHdnZ0VpCk1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRQ3JtaitTTmtGUHEvK2FXUFV1MlpFamtSK3AKTm1PeEVNSnZZcGhHNkJvRFAySE9ZbGRzdk9FWkRkbTBpWFlmeFIwZm5rUmtTMWEzSlZiYmhINWJnTElKb0dxcwo5aWpzN2hyQ0Rrdk9uRWxpUEZuc05pQ2NWNDNxNkZYaFMvNFpoNGpOMnlyUkU2UmZiS1BEeUw0a282NkFhSld1CnVkTldKVWpzSFZBSWowZHlnTXFKYm0rT29iSzk5ckUxcDg5Z3RNUStJdzFkWnUvUFF4SjlYOStMeXdxZUxPckQKOWhpNWkxajNFUUp2RXQxSVUzclEwc2E0NU5zZkt4YzEwZjdhTjJuSDQzSnhnMVRiZXNPOWYrcWlyeDBHYmVSYQpyVmNaazNVaFc2cHZmam9XbDBEc0NwNTJwZDBQN05rUmhmak44b2RMN0h3bFVIc1NqemlSYytsTG5YREJBZ01CCkFBRXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBSVdKdmRPZ01PUnQxWk53SENkNTNieTlkMlBkcW5tWHFZZ20KNDZHK2Fvb1dSeTJKMEMwS3ZOVGZGbEJFOUlydzNXUTVNMnpqY25qSUp5bzNLRUM5TDdPMnQ1WC9LTGVDck5ZVgpIc1d4cU5BTVBGY2VBa09HT0I1TThGVllkdjJTaVV2UDJjMEZQSzc2WFVzcVNkdnRsWGFkTk5ENzE3T0NTNm0yCnBIVjh1NWJNd1VmR2NCVFpEV2o4bjIzRVdHaXdnYkJkdDc3Z3h3YWc5NTROZkM2Ny9nSUc5ZlRrTTQ4aVJCUzEKc0NGYVBjMkFIT3hiMSs0ajVCMVY2Z29iZDZYWkFvbHdNaTNHUUtkbEM1NXZNeTNwK09WbDNNbEc3RWNTVUpMdApwZ2ZKaWw3L3dTWWhpUnhJU3hrYkk5cWhvNEwzZm5PZVB3clFVd2FzU1ZiL1lxbHZ2WHM9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" +) + +func TestReadVaultToken(t *testing.T) { + t.Run("read correct token", func(t *testing.T) { + dir := os.TempDir() + f, err := ioutil.TempFile(dir, "vault-token") + assert.NoError(t, err) + fileName := f.Name() + defer os.Remove(fileName) + + tokenString := "thisisnottheroottoken" + _, err = f.WriteString(tokenString) + assert.NoError(t, err) + + v := vaultSecretStore{ + vaultTokenMountPath: f.Name(), + } + + token, err := v.readVaultToken() + assert.Nil(t, err) + assert.Equal(t, tokenString, token) + }) + + t.Run("read incorrect token", func(t *testing.T) { + dir := os.TempDir() + f, err := ioutil.TempFile(dir, "vault-token") + assert.NoError(t, err) + fileName := f.Name() + defer os.Remove(fileName) + + tokenString := "thisisnottheroottoken" + _, err = f.WriteString(tokenString) + assert.NoError(t, err) + + v := vaultSecretStore{ + vaultTokenMountPath: f.Name(), + } + token, err := v.readVaultToken() + assert.Nil(t, err) + assert.NotEqual(t, "thisistheroottoken", token) + }) +} + +func TestVaultTLSConfig(t *testing.T) { + t.Run("with tls configuration", func(t *testing.T) { + certBytes := getCertificate() + properties := map[string]string{ + "caCert": string(certBytes), + "skipVerify": "false", + "tlsServerName": "vaultproject.io", + } + + m := secretstores.Metadata{ + Properties: properties, + } + + tlsConfig := metadataToTLSConfig(m.Properties) + skipVerify, err := strconv.ParseBool(properties["skipVerify"]) + assert.Nil(t, err) + assert.Equal(t, properties["caCert"], tlsConfig.vaultCACert) + assert.Equal(t, skipVerify, tlsConfig.vaultSkipVerify) + assert.Equal(t, properties["tlsServerName"], tlsConfig.vaultServerName) + }) +} + +func getCertificate() []byte { + certificateBytes, _ := base64.StdEncoding.DecodeString(certificate) + + return certificateBytes +}