docker
/
docs
mirror of https://github.com/docker/docs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adds specific memory backend support to server and signer. 

The server already supported a memory backend, but now it must be
specified, rather than just being a fallback if no storage is
specified.  This also adds a signer backend to signer, which
previously required a MySQL backend.

Thanks @endophage for the excellent suggestion!

Signed-off-by: Ying Li <ying.li@docker.com>
This commit is contained in:
Ying Li 2015-11-23 19:22:22 -05:00
parent f1bd28caf4
commit a94a47651f
6 changed files with 154 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cmd/notary-server/main.go
View File

@ -102,19 +102,19 @@ func getStore(configuration *viper.Viper, allowedBackends []string) (
if err != nil {
return nil, err
}
if storeConfig != nil {
logrus.Infof("Using %s backend", storeConfig.Backend)
store, err := storage.NewSQLStorage(storeConfig.Backend, storeConfig.Source)
if err != nil {
return nil, fmt.Errorf("Error starting DB driver: ", err.Error())
}
health.RegisterPeriodicFunc(
"DB operational", store.CheckHealth, time.Second*60)
return store, nil
logrus.Infof("Using %s backend", storeConfig.Backend)
if storeConfig.Backend == utils.MemoryBackend {
return storage.NewMemStorage(), nil
}
logrus.Debug("Using memory backend")
return storage.NewMemStorage(), nil
store, err := storage.NewSQLStorage(storeConfig.Backend, storeConfig.Source)
if err != nil {
return nil, fmt.Errorf("Error starting DB driver: ", err.Error())
}
health.RegisterPeriodicFunc(
"DB operational", store.CheckHealth, time.Second*60)
return store, nil
}
func main() {
@ -198,7 +198,7 @@ func main() {
trust = signed.NewEd25519()
}
store, err := getStore(mainViper, []string{"mysql"})
store, err := getStore(mainViper, []string{utils.MySQLBackend, utils.MemoryBackend})
if err != nil {
logrus.Fatal(err.Error())
}

12
cmd/notary-server/main_test.go
View File

@ -10,6 +10,7 @@ import (
"testing"
"github.com/docker/notary/server/storage"
"github.com/docker/notary/utils"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
)
@ -172,18 +173,19 @@ func TestGetStoreDBStore(t *testing.T) {
tmpFile.Close()
defer os.Remove(tmpFile.Name())
config := fmt.Sprintf(`{"storage": {"backend": "sqlite3", "db_url": "%s"}}`,
tmpFile.Name())
config := fmt.Sprintf(`{"storage": {"backend": "%s", "db_url": "%s"}}`,
utils.SqliteBackend, tmpFile.Name())
store, err := getStore(configure(config), []string{"sqlite3"})
store, err := getStore(configure(config), []string{utils.SqliteBackend})
assert.NoError(t, err)
_, ok := store.(*storage.SQLStorage)
assert.True(t, ok)
}
func TestGetMemoryStore(t *testing.T) {
config := fmt.Sprintf(`{"storage": {}}`)
store, err := getStore(configure(config), []string{"mysql"})
config := fmt.Sprintf(`{"storage": {"backend": "%s"}}`, utils.MemoryBackend)
store, err := getStore(configure(config),
[]string{utils.MySQLBackend, utils.MemoryBackend})
assert.NoError(t, err)
_, ok := store.(*storage.MemStorage)
assert.True(t, ok)

59
cmd/notary-signer/main.go
View File

@ -22,9 +22,11 @@ import (
"github.com/docker/distribution/health"
"github.com/docker/notary/cryptoservice"
"github.com/docker/notary/passphrase"
"github.com/docker/notary/signer"
"github.com/docker/notary/signer/api"
"github.com/docker/notary/signer/keydbstore"
"github.com/docker/notary/trustmanager"
"github.com/docker/notary/tuf/data"
"github.com/docker/notary/utils"
"github.com/docker/notary/version"
@ -75,43 +77,43 @@ func setUpCryptoservices(configuration *viper.Viper, allowedBackends []string) (
return nil, err
}
if storeConfig == nil {
return nil, fmt.Errorf("DB storage configuration is mandatory")
}
var keyStore trustmanager.KeyStore
if storeConfig.Backend == utils.MemoryBackend {
keyStore = trustmanager.NewKeyMemoryStore(
passphrase.ConstantRetriever("memory-db-ignore"))
} else {
defaultAlias := configuration.GetString("storage.default_alias")
if defaultAlias == "" {
// backwards compatibility - support this environment variable
defaultAlias = configuration.GetString(defaultAliasEnv)
}
dbSQL, err := sql.Open(storeConfig.Backend, storeConfig.Source)
if err != nil {
return nil, fmt.Errorf("failed to open the %s database: %s, %v",
storeConfig.Backend, storeConfig.Source, err)
}
logrus.Debugf("Using %s DB: %s", storeConfig.Backend, storeConfig.Source)
if defaultAlias == "" {
return nil, fmt.Errorf("must provide a default alias for the key DB")
}
logrus.Debug("Default Alias: ", defaultAlias)
defaultAlias := configuration.GetString("storage.default_alias")
if defaultAlias == "" {
// backwards compatibility - support this environment variable
defaultAlias = configuration.GetString(defaultAliasEnv)
}
dbSQL, err := sql.Open(storeConfig.Backend, storeConfig.Source)
if err != nil {
return nil, fmt.Errorf("failed to open the %s database: %s, %v",
storeConfig.Backend, storeConfig.Source, err)
}
logrus.Debugf("Using %s DB: %s", storeConfig.Backend, storeConfig.Source)
if defaultAlias == "" {
return nil, fmt.Errorf("must provide a default alias for the key DB")
}
logrus.Debug("Default Alias: ", defaultAlias)
keyStore, err := keydbstore.NewKeyDBStore(
passphraseRetriever, defaultAlias, storeConfig.Backend, dbSQL)
if err != nil {
return nil, fmt.Errorf("failed to create a new keydbstore: %v", err)
}
keyStore, err := keydbstore.NewKeyDBStore(
passphraseRetriever, defaultAlias, storeConfig.Backend, dbSQL)
if err != nil {
return nil, fmt.Errorf("failed to create a new keydbstore: %v", err)
health.RegisterPeriodicFunc(
"DB operational", keyStore.HealthCheck, time.Second*60)
}
health.RegisterPeriodicFunc(
"DB operational", keyStore.HealthCheck, time.Second*60)
cryptoService := cryptoservice.NewCryptoService("", keyStore)
cryptoServices := make(signer.CryptoServiceIndex)
cryptoServices[data.ED25519Key] = cryptoService
cryptoServices[data.ECDSAKey] = cryptoService
return cryptoServices, nil
}
@ -220,7 +222,8 @@ func main() {
}
// setup the cryptoservices
cryptoServices, err := setUpCryptoservices(mainViper, []string{"mysql"})
cryptoServices, err := setUpCryptoservices(mainViper,
[]string{utils.MySQLBackend, utils.MemoryBackend})
if err != nil {
logrus.Fatal(err.Error())
}

48
cmd/notary-signer/main_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/docker/notary/signer"
"github.com/docker/notary/tuf/data"
"github.com/docker/notary/utils"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
)
@ -30,6 +31,10 @@ func configure(jsonConfig string) *viper.Viper {
return config
}
// If the TLS configuration is invalid, an error is returned. This doesn't test
// all the cases of the TLS configuration being invalid, since it's just
// calling configuration.ParseTLSConfig - this test just makes sure the
// error is propogated.
func TestGetAddrAndTLSConfigInvalidTLS(t *testing.T) {
invalids := []string{
`{"server": {"http_addr": ":1234", "grpc_addr": ":2345"}}`,
@ -47,6 +52,7 @@ func TestGetAddrAndTLSConfigInvalidTLS(t *testing.T) {
}
}
// If a GRPC address is not provided, an error is returned.
func TestGetAddrAndTLSConfigNoGRPCAddr(t *testing.T) {
_, _, _, err := getAddrAndTLSConfig(configure(fmt.Sprintf(`{
"server": {
@ -59,6 +65,7 @@ func TestGetAddrAndTLSConfigNoGRPCAddr(t *testing.T) {
assert.Contains(t, err.Error(), "grpc listen address required for server")
}
// If an HTTP address is not provided, an error is returned.
func TestGetAddrAndTLSConfigNoHTTPAddr(t *testing.T) {
_, _, _, err := getAddrAndTLSConfig(configure(fmt.Sprintf(`{
"server": {
@ -71,6 +78,7 @@ func TestGetAddrAndTLSConfigNoHTTPAddr(t *testing.T) {
assert.Contains(t, err.Error(), "http listen address required for server")
}
// Success parsing a valid TLS config, HTTP address, and GRPC address.
func TestGetAddrAndTLSConfigSuccess(t *testing.T) {
httpAddr, grpcAddr, tlsConf, err := getAddrAndTLSConfig(configure(fmt.Sprintf(`{
"server": {
@ -86,7 +94,8 @@ func TestGetAddrAndTLSConfigSuccess(t *testing.T) {
assert.NotNil(t, tlsConf)
}
func TestSetupCryptoServicesNoDefaultAlias(t *testing.T) {
// If a default alias is not provided to a DB backend, an error is returned.
func TestSetupCryptoServicesDBStoreNoDefaultAlias(t *testing.T) {
tmpFile, err := ioutil.TempFile("/tmp", "sqlite3")
assert.NoError(t, err)
tmpFile.Close()
@ -94,14 +103,18 @@ func TestSetupCryptoServicesNoDefaultAlias(t *testing.T) {
_, err = setUpCryptoservices(
configure(fmt.Sprintf(
`{"storage": {"backend": "sqlite3", "db_url": "%s"}}`,
tmpFile.Name())),
[]string{"sqlite3"})
`{"storage": {"backend": "%s", "db_url": "%s"}}`,
utils.SqliteBackend, tmpFile.Name())),
[]string{utils.SqliteBackend})
assert.Error(t, err)
assert.Contains(t, err.Error(), "must provide a default alias for the key DB")
}
func TestSetupCryptoServicesSuccess(t *testing.T) {
// If a default alias *is* provided to a valid DB backend, a valid
// CryptoService is returned. (This depends on ParseStorage, which is tested
// separately, so this doesn't test all the possible cases of storage
// success/failure).
func TestSetupCryptoServicesDBStoreSuccess(t *testing.T) {
tmpFile, err := ioutil.TempFile("/tmp", "sqlite3")
assert.NoError(t, err)
tmpFile.Close()
@ -109,10 +122,29 @@ func TestSetupCryptoServicesSuccess(t *testing.T) {
cryptoServices, err := setUpCryptoservices(
configure(fmt.Sprintf(
`{"storage": {"backend": "sqlite3", "db_url": "%s"},
`{"storage": {"backend": "%s", "db_url": "%s"},
"default_alias": "timestamp"}`,
tmpFile.Name())),
[]string{"sqlite3"})
utils.SqliteBackend, tmpFile.Name())),
[]string{utils.SqliteBackend})
assert.NoError(t, err)
assert.Len(t, cryptoServices, 2)
edService, ok := cryptoServices[data.ED25519Key]
assert.True(t, ok)
ecService, ok := cryptoServices[data.ECDSAKey]
assert.True(t, ok)
assert.Equal(t, edService, ecService)
}
// If a memory backend is specified, then a default alias is not needed, and
// a valid CryptoService is returned.
func TestSetupCryptoServicesMemoryStore(t *testing.T) {
config := configure(fmt.Sprintf(`{"storage": {"backend": "%s"}}`,
utils.MemoryBackend))
cryptoServices, err := setUpCryptoservices(config,
[]string{utils.SqliteBackend, utils.MemoryBackend})
assert.NoError(t, err)
assert.Len(t, cryptoServices, 2)

43
utils/configuration.go
View File

@ -13,6 +13,13 @@ import (
"github.com/spf13/viper"
)
// Specifies the list of recognized backends
const (
MemoryBackend = "memory"
MySQLBackend = "mysql"
SqliteBackend = "sqlite3"
)
// Storage is a configuration about what storage backend a server should use
type Storage struct {
Backend string
@ -77,29 +84,37 @@ func ParseLogLevel(configuration *viper.Viper, defaultLevel logrus.Level) (
}
// ParseStorage tries to parse out Storage from a Viper. If backend and
// URL are not provided, returns a nil pointer.
func ParseStorage(configuration *viper.Viper, allowedBackeneds []string) (*Storage, error) {
// URL are not provided, returns a nil pointer. Storage is required (if
// a backend is not provided, an error will be returned.)
func ParseStorage(configuration *viper.Viper, allowedBackends []string) (*Storage, error) {
store := Storage{
Backend: configuration.GetString("storage.backend"),
Source: configuration.GetString("storage.db_url"),
}
if store.Backend == "" && store.Source == "" {
return nil, nil
}
if store.Source == "" {
return nil, fmt.Errorf("must provide a non-empty database source")
}
supported := false
store.Backend = strings.ToLower(store.Backend)
for _, backend := range allowedBackeneds {
for _, backend := range allowedBackends {
if backend == store.Backend {
return &store, nil
supported = true
break
}
}
return nil, fmt.Errorf(
"must specify one of these supported backends: %s",
strings.Join(allowedBackeneds, ", "))
if !supported {
return nil, fmt.Errorf(
"must specify one of these supported backends: %s",
strings.Join(allowedBackends, ", "))
}
if store.Backend == MemoryBackend {
return &Storage{Backend: MemoryBackend}, nil
}
if store.Source == "" {
return nil, fmt.Errorf(
"must provide a non-empty database source for %s", store.Backend)
}
return &store, nil
}
// ParseBugsnag tries to parse out a Bugsnag Configuration from a Viper.

55
utils/configuration_test.go
View File

@ -4,7 +4,6 @@ import (
"bytes"
"fmt"
"os"
"strings"
"testing"
"github.com/Sirupsen/logrus"
@ -146,38 +145,54 @@ func TestParseBugsnagWithEnvironmentVariables(t *testing.T) {
assert.Equal(t, expected, *bugconf)
}
// If the storage parameters are invalid, an error is returned
func TestParseInvalidStorage(t *testing.T) {
// If the storage backend is invalid or not provided, an error is returned.
func TestParseInvalidStorageBackend(t *testing.T) {
invalids := []string{
`{"storage": {"backend": "postgres", "db_url": "1234"}}`,
`{"storage": {"db_url": "12345"}}`,
`{"storage": {"backend": "mysql"}}`,
`{"storage": {"backend": "sqlite3", "db_url": ""}}`,
`{"storage": {}}`,
`{}`,
}
for _, configJSON := range invalids {
_, err := ParseStorage(configure(configJSON), []string{"mysql", "sqlite3"})
_, err := ParseStorage(configure(configJSON),
[]string{MySQLBackend, SqliteBackend})
assert.Error(t, err, fmt.Sprintf("'%s' should be an error", configJSON))
if strings.Contains(configJSON, "mysql") || strings.Contains(configJSON, "sqlite3") {
assert.Contains(t, err.Error(),
"must specify one of these supported backends: mysql, sqlite3")
}
}
// If there is no DB url for non-memory backends, an error is returned.
func TestParseInvalidStorageNoDBSource(t *testing.T) {
invalids := []string{
`{"storage": {"backend": "%s"}}`,
`{"storage": {"backend": "%s", "db_url": ""}}`,
}
for _, backend := range []string{MySQLBackend, SqliteBackend} {
for _, configJSONFmt := range invalids {
configJSON := fmt.Sprintf(configJSONFmt, backend)
_, err := ParseStorage(configure(configJSON),
[]string{MySQLBackend, SqliteBackend})
assert.Error(t, err, fmt.Sprintf("'%s' should be an error", configJSON))
assert.Contains(t, err.Error(),
"must provide a non-empty database source")
} else {
assert.Contains(t, err.Error(),
"must specify one of these supported backends: mysql, sqlite3")
fmt.Sprintf("must provide a non-empty database source for %s", backend))
}
}
}
// If there is no storage, a nil pointer is returned
func TestParseNoStorage(t *testing.T) {
empties := []string{`{}`, `{"storage": {}}`}
for _, configJSON := range empties {
store, err := ParseStorage(configure(configJSON), []string{"mysql"})
assert.NoError(t, err)
assert.Nil(t, store)
}
// If a memory storage backend is specified, no DB URL is necessary for a
// successful storage parse.
func TestParseStorageMemoryStore(t *testing.T) {
config := configure(`{"storage": {"backend": "MEMORY"}}`)
expected := Storage{Backend: MemoryBackend}
store, err := ParseStorage(config, []string{MySQLBackend, MemoryBackend})
assert.NoError(t, err)
assert.Equal(t, expected, *store)
}
func TestParseStorage(t *testing.T) {
// A supported backend with DB source will be successfully parsed.
func TestParseStorageDBStore(t *testing.T) {
config := configure(`{
"storage": {
"backend": "MySQL",