From 155131a3646d957ba5993b11a10a0e38f4ad1cd4 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 17 Dec 2014 16:14:56 -0800 Subject: [PATCH 1/4] Add a simple key<->Container persistent store. Signed-off-by: Andrea Luzzardi --- cluster/store.go | 177 ++++++++++++++++++++++++++++++++++++++++++ cluster/store_test.go | 47 +++++++++++ 2 files changed, 224 insertions(+) create mode 100644 cluster/store.go create mode 100644 cluster/store_test.go diff --git a/cluster/store.go b/cluster/store.go new file mode 100644 index 0000000000..b021923b20 --- /dev/null +++ b/cluster/store.go @@ -0,0 +1,177 @@ +package cluster + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "sync" +) + +var ( + ErrNotFound = errors.New("not found") + ErrAlreadyExists = errors.New("already exists") +) + +// A simple key<->Container store. +type Store struct { + RootDir string + containers map[string]*Container + + sync.RWMutex +} + +func NewStore(rootdir string) *Store { + return &Store{ + RootDir: rootdir, + containers: make(map[string]*Container), + } +} + +// Initialize must be called before performing any operation on the store. It +// will attempt to restore the data from disk. +func (s *Store) Initialize() error { + s.Lock() + defer s.Unlock() + + if err := os.MkdirAll(s.RootDir, 0700); err != nil && !os.IsNotExist(err) { + return err + } + + if err := s.restore(); err != nil { + return err + } + + return nil +} + +func (s *Store) path(key string) string { + return path.Join(s.RootDir, key+".json") +} + +func (s *Store) restore() error { + files, err := ioutil.ReadDir(s.RootDir) + if err != nil { + return err + } + for _, fileinfo := range files { + file := fileinfo.Name() + + // Verify the file extension. + extension := filepath.Ext(file) + if extension != ".json" { + return fmt.Errorf("invalid file extension for filename %s (%s)", file, extension) + } + + // Load the object back. + container, err := s.load(path.Join(s.RootDir, file)) + if err != nil { + return err + } + + // Extract the key. + key := file[0 : len(file)-len(extension)] + if len(key) == 0 { + return fmt.Errorf("invalid filename %s", file) + } + + // Store it back. + s.containers[key] = container + } + return nil +} + +func (s *Store) load(file string) (*Container, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("unable to load %s: %v", file, err) + } + container := &Container{} + if err := json.Unmarshal(data, container); err != nil { + return nil, err + } + return container, nil +} + +// Retrieves an object from the store keyed by `key`. +func (s *Store) Get(key string) (*Container, error) { + s.RLock() + defer s.RUnlock() + + if value, ok := s.containers[key]; ok { + return value, nil + } + return nil, ErrNotFound +} + +// Return all objects of the store. +func (s *Store) All() []*Container { + s.RLock() + defer s.RUnlock() + + states := make([]*Container, len(s.containers)) + i := 0 + for _, state := range s.containers { + states[i] = state + i = i + 1 + } + return states +} + +func (s *Store) set(key string, value *Container) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + + if err := ioutil.WriteFile(s.path(key), data, 0600); err != nil { + return err + } + + s.containers[key] = value + return nil +} + +// Add a new object on the store. `key` must be unique. +func (s *Store) Add(key string, value *Container) error { + s.Lock() + defer s.Unlock() + + if _, exists := s.containers[key]; exists { + return ErrAlreadyExists + } + + return s.set(key, value) +} + +// Replaces an already existing object from the store. +func (s *Store) Replace(key string, value *Container) error { + s.Lock() + defer s.Unlock() + + if _, exists := s.containers[key]; !exists { + return ErrNotFound + } + + return s.set(key, value) +} + +// Remove `key` from the store. +func (s *Store) Remove(key string) error { + s.Lock() + defer s.Unlock() + + if _, exists := s.containers[key]; !exists { + return ErrNotFound + } + + if err := os.Remove(s.path(key)); err != nil { + return err + } + + delete(s.containers, key) + return nil +} diff --git a/cluster/store_test.go b/cluster/store_test.go new file mode 100644 index 0000000000..0cad4fdc43 --- /dev/null +++ b/cluster/store_test.go @@ -0,0 +1,47 @@ +package cluster + +import ( + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStore(t *testing.T) { + dir, err := ioutil.TempDir("", "store-test") + assert.NoError(t, err) + store := NewStore(dir) + assert.NoError(t, store.Initialize()) + + c1 := &Container{} + c1.Id = "foo" + c2 := &Container{} + c2.Id = "bar" + + var ret *Container + + // Add "foo" into the store. + assert.NoError(t, store.Add("foo", c1)) + + // Retrieve "foo" from the store. + ret, err = store.Get("foo") + assert.NoError(t, err) + assert.Equal(t, c1.Id, ret.Id) + + // Try to add "foo" again. + assert.EqualError(t, store.Add("foo", c1), ErrAlreadyExists.Error()) + + // Replace "foo" with c2. + assert.NoError(t, store.Replace("foo", c2)) + ret, err = store.Get("foo") + assert.NoError(t, err) + assert.Equal(t, c2.Id, ret.Id) + + // Initialize a brand new store and retrieve "foo" again. + // This is to ensure data load on initialization works correctly. + store = NewStore(dir) + assert.NoError(t, store.Initialize()) + ret, err = store.Get("foo") + assert.NoError(t, err) + assert.Equal(t, c2.Id, ret.Id) +} From 410e7969b795656a6e36dd90cfece9a4e7a837bc Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 12 Jan 2015 09:10:06 -0800 Subject: [PATCH 2/4] Store: Store the RequestedState rather than the entire Container. Signed-off-by: Andrea Luzzardi --- cluster/store.go | 55 ++++++++++++++++++++++++------------------- cluster/store_test.go | 16 ++++++------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/cluster/store.go b/cluster/store.go index b021923b20..fd4b2158c2 100644 --- a/cluster/store.go +++ b/cluster/store.go @@ -9,6 +9,8 @@ import ( "path" "path/filepath" "sync" + + "github.com/samalba/dockerclient" ) var ( @@ -16,18 +18,23 @@ var ( ErrAlreadyExists = errors.New("already exists") ) -// A simple key<->Container store. +// A simple key<->RequestedState store. type Store struct { - RootDir string - containers map[string]*Container + RootDir string + values map[string]*RequestedState sync.RWMutex } +type RequestedState struct { + Name string + Config *dockerclient.ContainerConfig +} + func NewStore(rootdir string) *Store { return &Store{ - RootDir: rootdir, - containers: make(map[string]*Container), + RootDir: rootdir, + values: make(map[string]*RequestedState), } } @@ -67,7 +74,7 @@ func (s *Store) restore() error { } // Load the object back. - container, err := s.load(path.Join(s.RootDir, file)) + value, err := s.load(path.Join(s.RootDir, file)) if err != nil { return err } @@ -79,49 +86,49 @@ func (s *Store) restore() error { } // Store it back. - s.containers[key] = container + s.values[key] = value } return nil } -func (s *Store) load(file string) (*Container, error) { +func (s *Store) load(file string) (*RequestedState, error) { data, err := ioutil.ReadFile(file) if err != nil { return nil, fmt.Errorf("unable to load %s: %v", file, err) } - container := &Container{} - if err := json.Unmarshal(data, container); err != nil { + value := &RequestedState{} + if err := json.Unmarshal(data, value); err != nil { return nil, err } - return container, nil + return value, nil } // Retrieves an object from the store keyed by `key`. -func (s *Store) Get(key string) (*Container, error) { +func (s *Store) Get(key string) (*RequestedState, error) { s.RLock() defer s.RUnlock() - if value, ok := s.containers[key]; ok { + if value, ok := s.values[key]; ok { return value, nil } return nil, ErrNotFound } // Return all objects of the store. -func (s *Store) All() []*Container { +func (s *Store) All() []*RequestedState { s.RLock() defer s.RUnlock() - states := make([]*Container, len(s.containers)) + states := make([]*RequestedState, len(s.values)) i := 0 - for _, state := range s.containers { + for _, state := range s.values { states[i] = state i = i + 1 } return states } -func (s *Store) set(key string, value *Container) error { +func (s *Store) set(key string, value *RequestedState) error { data, err := json.MarshalIndent(value, "", " ") if err != nil { return err @@ -131,16 +138,16 @@ func (s *Store) set(key string, value *Container) error { return err } - s.containers[key] = value + s.values[key] = value return nil } // Add a new object on the store. `key` must be unique. -func (s *Store) Add(key string, value *Container) error { +func (s *Store) Add(key string, value *RequestedState) error { s.Lock() defer s.Unlock() - if _, exists := s.containers[key]; exists { + if _, exists := s.values[key]; exists { return ErrAlreadyExists } @@ -148,11 +155,11 @@ func (s *Store) Add(key string, value *Container) error { } // Replaces an already existing object from the store. -func (s *Store) Replace(key string, value *Container) error { +func (s *Store) Replace(key string, value *RequestedState) error { s.Lock() defer s.Unlock() - if _, exists := s.containers[key]; !exists { + if _, exists := s.values[key]; !exists { return ErrNotFound } @@ -164,7 +171,7 @@ func (s *Store) Remove(key string) error { s.Lock() defer s.Unlock() - if _, exists := s.containers[key]; !exists { + if _, exists := s.values[key]; !exists { return ErrNotFound } @@ -172,6 +179,6 @@ func (s *Store) Remove(key string) error { return err } - delete(s.containers, key) + delete(s.values, key) return nil } diff --git a/cluster/store_test.go b/cluster/store_test.go index 0cad4fdc43..7e05deda7c 100644 --- a/cluster/store_test.go +++ b/cluster/store_test.go @@ -13,12 +13,12 @@ func TestStore(t *testing.T) { store := NewStore(dir) assert.NoError(t, store.Initialize()) - c1 := &Container{} - c1.Id = "foo" - c2 := &Container{} - c2.Id = "bar" + c1 := &RequestedState{} + c1.Name = "foo" + c2 := &RequestedState{} + c2.Name = "bar" - var ret *Container + var ret *RequestedState // Add "foo" into the store. assert.NoError(t, store.Add("foo", c1)) @@ -26,7 +26,7 @@ func TestStore(t *testing.T) { // Retrieve "foo" from the store. ret, err = store.Get("foo") assert.NoError(t, err) - assert.Equal(t, c1.Id, ret.Id) + assert.Equal(t, c1.Name, ret.Name) // Try to add "foo" again. assert.EqualError(t, store.Add("foo", c1), ErrAlreadyExists.Error()) @@ -35,7 +35,7 @@ func TestStore(t *testing.T) { assert.NoError(t, store.Replace("foo", c2)) ret, err = store.Get("foo") assert.NoError(t, err) - assert.Equal(t, c2.Id, ret.Id) + assert.Equal(t, c2.Name, ret.Name) // Initialize a brand new store and retrieve "foo" again. // This is to ensure data load on initialization works correctly. @@ -43,5 +43,5 @@ func TestStore(t *testing.T) { assert.NoError(t, store.Initialize()) ret, err = store.Get("foo") assert.NoError(t, err) - assert.Equal(t, c2.Id, ret.Id) + assert.Equal(t, c2.Name, ret.Name) } From 7cc70ee2764e42cdbaec50c1ac629e87f0cdf44d Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 12 Jan 2015 09:13:29 -0800 Subject: [PATCH 3/4] Store: Move the store into the state package. Signed-off-by: Andrea Luzzardi --- state/state.go | 10 ++++++++++ {cluster => state}/store.go | 9 +-------- {cluster => state}/store_test.go | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 state/state.go rename {cluster => state}/store.go (96%) rename {cluster => state}/store_test.go (98%) diff --git a/state/state.go b/state/state.go new file mode 100644 index 0000000000..10173255e5 --- /dev/null +++ b/state/state.go @@ -0,0 +1,10 @@ +package state + +import ( + "github.com/samalba/dockerclient" +) + +type RequestedState struct { + Name string + Config *dockerclient.ContainerConfig +} diff --git a/cluster/store.go b/state/store.go similarity index 96% rename from cluster/store.go rename to state/store.go index fd4b2158c2..89526f0bb5 100644 --- a/cluster/store.go +++ b/state/store.go @@ -1,4 +1,4 @@ -package cluster +package state import ( "encoding/json" @@ -9,8 +9,6 @@ import ( "path" "path/filepath" "sync" - - "github.com/samalba/dockerclient" ) var ( @@ -26,11 +24,6 @@ type Store struct { sync.RWMutex } -type RequestedState struct { - Name string - Config *dockerclient.ContainerConfig -} - func NewStore(rootdir string) *Store { return &Store{ RootDir: rootdir, diff --git a/cluster/store_test.go b/state/store_test.go similarity index 98% rename from cluster/store_test.go rename to state/store_test.go index 7e05deda7c..78b28edc83 100644 --- a/cluster/store_test.go +++ b/state/store_test.go @@ -1,4 +1,4 @@ -package cluster +package state import ( "io/ioutil" From 750ab8f60fb5957a199846e24d6f95529133bf6b Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 14 Jan 2015 08:35:07 -0800 Subject: [PATCH 4/4] Store: Improve error handling. Signed-off-by: Andrea Luzzardi --- state/store.go | 16 +++++++++++++--- state/store_test.go | 3 +++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/state/store.go b/state/store.go index 89526f0bb5..224d501330 100644 --- a/state/store.go +++ b/state/store.go @@ -9,11 +9,14 @@ import ( "path" "path/filepath" "sync" + + log "github.com/Sirupsen/logrus" ) var ( ErrNotFound = errors.New("not found") ErrAlreadyExists = errors.New("already exists") + ErrInvalidKey = errors.New("invalid key") ) // A simple key<->RequestedState store. @@ -63,19 +66,22 @@ func (s *Store) restore() error { // Verify the file extension. extension := filepath.Ext(file) if extension != ".json" { - return fmt.Errorf("invalid file extension for filename %s (%s)", file, extension) + log.Errorf("invalid file extension for filename %s (%s)", file, extension) + continue } // Load the object back. value, err := s.load(path.Join(s.RootDir, file)) if err != nil { - return err + log.Errorf(err.Error()) + continue } // Extract the key. key := file[0 : len(file)-len(extension)] if len(key) == 0 { - return fmt.Errorf("invalid filename %s", file) + log.Errorf("invalid filename %s", file) + continue } // Store it back. @@ -122,6 +128,10 @@ func (s *Store) All() []*RequestedState { } func (s *Store) set(key string, value *RequestedState) error { + if len(key) == 0 { + return ErrInvalidKey + } + data, err := json.MarshalIndent(value, "", " ") if err != nil { return err diff --git a/state/store_test.go b/state/store_test.go index 78b28edc83..7eb379cb91 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -20,6 +20,9 @@ func TestStore(t *testing.T) { var ret *RequestedState + // Add an invalid key + assert.EqualError(t, store.Add("", c1), ErrInvalidKey.Error()) + // Add "foo" into the store. assert.NoError(t, store.Add("foo", c1))