From 902d660d2ff48727386718440162ed2a8399eaf0 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 15 Feb 2013 18:46:46 -0800 Subject: [PATCH 1/5] docker/fs: a robust filesytem store --- fs/store.go | 156 ++++++++++++++++++++++++++++++++++ fs/store_test.go | 217 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 373 insertions(+) create mode 100644 fs/store.go create mode 100644 fs/store_test.go diff --git a/fs/store.go b/fs/store.go new file mode 100644 index 0000000000..e429597621 --- /dev/null +++ b/fs/store.go @@ -0,0 +1,156 @@ +package fs + +import ( + "database/sql" + _ "github.com/mattn/go-sqlite3" + "github.com/coopernurse/gorp" + "os" + "io" + "path" + "github.com/dotcloud/docker/future" +) + +type Store struct { + Root string + db *sql.DB + orm *gorp.DbMap +} + +type Archive io.Reader + +func New(root string) (*Store, error) { + if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + return nil, err + } + db, err := sql.Open("sqlite3", path.Join(root, "db")) + if err != nil { + return nil, err + } + orm := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} + orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id") + orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") + if err := orm.CreateTables(); err != nil { + return nil, err + } + return &Store{ + Root: root, + db: db, + orm: orm, + }, nil +} + +func (store *Store) imageList(src []interface{}) ([]*Image) { + var images []*Image + for _, i := range src { + img := i.(*Image) + img.store = store + images = append(images, img) + } + return images +} + +func (store *Store) Images() ([]*Image, error) { + images , err := store.orm.Select(Image{}, "select * from images") + if err != nil { + return nil, err + } + return store.imageList(images), nil +} + +func (store *Store) Paths() ([]string, error) { + var paths []string + rows, err := store.db.Query("select distinct Path from paths order by Path") + if err != nil { + return nil, err + } + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + paths = append(paths, path) + } + return paths, nil +} + +func (store *Store) List(pth string) ([]*Image, error) { + pth = path.Clean(pth) + images, err := store.orm.Select(Image{}, "select images.* from images, paths where Path=? and paths.Image=images.Id", pth) + if err != nil { + return nil, err + } + return store.imageList(images), nil +} + +func (store *Store) Get(id string) (*Image, error) { + images, err := store.orm.Select(Image{}, "select * from images where Id=?", id) + if err != nil { + return nil, err + } + if len(images) < 1 { + return nil, os.ErrNotExist + } + return images[0].(*Image), nil +} + +func (store *Store) Create(layer Archive, parent *Image, pth, comment string) (*Image, error) { + // FIXME: actually do something with the layer... + img := &Image{ + Id : future.RandomId(), + Comment: comment, + store: store, + } + path := &Path{ + Path: path.Clean(pth), + Image: img.Id, + } + trans, err := store.orm.Begin() + if err != nil { + return nil, err + } + if err := trans.Insert(img); err != nil { + return nil, err + } + if err := trans.Insert(path); err != nil { + return nil, err + } + if err := trans.Commit(); err != nil { + return nil, err + } + return img, nil +} + +func (store *Store) Register(image *Image, pth string) error { + // FIXME: import layer + trans, err := store.orm.Begin() + if err != nil { + return err + } + trans.Insert(image) + trans.Insert(&Path{Path: pth, Image: image.Id}) + return trans.Commit() +} + + + + +type Image struct { + Id string + Parent string + Comment string + store *Store `db:"-"` +} + + +func (image *Image) Copy(pth string) (*Image, error) { + if err := image.store.orm.Insert(&Path{Path: pth, Image: image.Id}); err != nil { + return nil, err + } + return image, nil +} + + +type Path struct { + Path string + Image string +} diff --git a/fs/store_test.go b/fs/store_test.go new file mode 100644 index 0000000000..940396227d --- /dev/null +++ b/fs/store_test.go @@ -0,0 +1,217 @@ +package fs + +import ( + "testing" + "io/ioutil" + "github.com/dotcloud/docker/fake" + "os" + "errors" + "fmt" +) + +func TestInit(t *testing.T) { + store, err := TempStore("testinit") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + paths, err := store.Paths() + if err != nil { + t.Fatal(err) + } + if l := len(paths); l != 0 { + t.Fatal("Fresh store should be empty after init (len=%d)", l) + } +} + +func TestCreate(t *testing.T) { + store, err := TempStore("testcreate") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + if images, err := store.Images(); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) + } + if images, err := store.List("foo"); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Path foo has wrong number of images (should be %d, not %d)", 1, l) + } else if images[0].Id != image.Id { + t.Fatalf("Imported image should be listed at path foo (%s != %s)", images[0], image) + } +} + +// Copy an image to a new path +func TestCopyNewPath(t *testing.T) { + store, err := TempStore("testcopynewpath") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + src, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + dst, err := src.Copy("bar") + if err != nil { + t.Fatal(err) + } + // ID should be the same + if src.Id != dst.Id { + t.Fatal("Different IDs") + } + // Check number of images at source path + if images, err := store.List("foo"); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatal("Wrong number of images at source path (should be %d, not %d)", 1, l) + } + // Check number of images at destination path + if images, err := store.List("bar"); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatal("Wrong number of images at destination path (should be %d, not %d)", 1, l) + } + if err := healthCheck(store); err != nil { + t.Fatal(err) + } +} + +// Copying an image to the same path twice should fail +func TestCopySameName(t *testing.T) { + store, err := TempStore("testcopysamename") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + src, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + _, err = src.Copy("foo") + if err == nil { + t.Fatal("Copying an image to the same patch twice should fail.") + } +} + +/* +func TestMount(t *testing.T) { + store, err := TempStore() + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + // Create mount targets + root, err := ioutil.TempDir("", "docker-fs-test") + if err != nil { + t.Fatal(err) + } + rw, err := ioutil.TempDir("", "docker-fs-test") + if err != nil { + t.Fatal(err) + } + mountpoint, err := image.Mount(root, rw) + if err != nil { + t.Fatal(err) + } + defer mountpoint.Umount() + // Mountpoint should be marked as mounted + if !mountpoint.Mounted() { + t.Fatal("Mountpoint not mounted") + } + // There should be one mountpoint registered + if l := len(image.Mountpoints()); l != 1 { + t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 1, l) + } + // Unmounting should work + if err := mountpoint.Umount(); err != nil { + t.Fatal(err) + } + // De-registering should work + if err := mountpoint.Deregister(); err != nil { + t.Fatal(err) + } + if l := len(image.Mountpoints()); l != 0 { + t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 0, l) + } + // General health check + if err := healthCheck(); err != nil { + t.Fatal(err) + } +} +*/ + +func TempStore(prefix string) (*Store, error) { + dir, err := ioutil.TempDir("", "docker-fs-test-" + prefix) + if err != nil { + return nil, err + } + return New(dir) +} + +func nuke(store *Store) error { + return os.RemoveAll(store.Root) +} + +// Look for inconsistencies in a store. +func healthCheck(store *Store) error { + parents := make(map[string]bool) + paths, err := store.Paths() + if err != nil { + return err + } + for _, path := range paths { + images, err := store.List(path) + if err != nil { + return err + } + IDs := make(map[string]bool) // All IDs for this path + for _, img := range images { + // Check for duplicate IDs per path + if _, exists := IDs[img.Id]; exists { + return errors.New(fmt.Sprintf("Duplicate ID: %s", img.Id)) + } else { + IDs[img.Id] = true + } + // Store parent for 2nd pass + if parent := img.Parent; parent != "" { + parents[parent] = true + } + } + } + // Check non-existing parents + for parent := range parents { + if _, exists := parents[parent]; !exists { + return errors.New("Reference to non-registered parent: " + parent) + } + } + return nil +} + From 07b6bc3fc738ab3a27e9760b0efc57c78d7f98af Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 15 Feb 2013 21:48:04 -0800 Subject: [PATCH 2/5] fs.store.Get(): use gorp.Get() instead of gorp.Select() --- fs/store.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/fs/store.go b/fs/store.go index e429597621..c34dbd0b40 100644 --- a/fs/store.go +++ b/fs/store.go @@ -83,14 +83,8 @@ func (store *Store) List(pth string) ([]*Image, error) { } func (store *Store) Get(id string) (*Image, error) { - images, err := store.orm.Select(Image{}, "select * from images where Id=?", id) - if err != nil { - return nil, err - } - if len(images) < 1 { - return nil, os.ErrNotExist - } - return images[0].(*Image), nil + img, err := store.orm.Get(Image{}, id) + return img.(*Image), err } func (store *Store) Create(layer Archive, parent *Image, pth, comment string) (*Image, error) { From aab32e1012bd6061ec1049adc2bbe60ed1918f54 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 15 Feb 2013 21:50:59 -0800 Subject: [PATCH 3/5] Fix store.Register() --- fs/store.go | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/store.go b/fs/store.go index c34dbd0b40..02bfd47e8f 100644 --- a/fs/store.go +++ b/fs/store.go @@ -115,6 +115,7 @@ func (store *Store) Create(layer Archive, parent *Image, pth, comment string) (* } func (store *Store) Register(image *Image, pth string) error { + image.store = store // FIXME: import layer trans, err := store.orm.Begin() if err != nil { From 1531848ca6ead7c46497a625feba1224825e9011 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 15 Feb 2013 21:51:36 -0800 Subject: [PATCH 4/5] Getting started with mountpoints --- fs/store.go | 27 ++++++++++++++++++++++++++ fs/store_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/fs/store.go b/fs/store.go index 02bfd47e8f..4ba23ae7dd 100644 --- a/fs/store.go +++ b/fs/store.go @@ -29,6 +29,7 @@ func New(root string) (*Store, error) { orm := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id") orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") + orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root") if err := orm.CreateTables(); err != nil { return nil, err } @@ -144,6 +145,32 @@ func (image *Image) Copy(pth string) (*Image, error) { return image, nil } +type Mountpoint struct { + Image string + Root string + Rw string +} + +func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { + mountpoint := &Mountpoint{Root: path.Clean(root), Rw: path.Clean(rw), Image: image.Id} + if err := image.store.orm.Insert(mountpoint); err != nil { + return nil, err + } + return mountpoint, nil +} + +func (image *Image) Mountpoints() ([]*Mountpoint, error) { + var mountpoints []*Mountpoint + res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=?", image.Id) + if err != nil { + return nil, err + } + for _, mp := range res { + mountpoints = append(mountpoints, mp.(*Mountpoint)) + } + return mountpoints, nil +} + type Path struct { Path string diff --git a/fs/store_test.go b/fs/store_test.go index 940396227d..89198f94d5 100644 --- a/fs/store_test.go +++ b/fs/store_test.go @@ -113,6 +113,55 @@ func TestCopySameName(t *testing.T) { } } +func TestMountPoint(t *testing.T) { + store, err := TempStore("test-mountpoint") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + mountpoint, err := image.Mountpoint("/tmp/a", "/tmp/b") + if err != nil { + t.Fatal(err) + } + if mountpoint.Root != "/tmp/a" { + t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/a", mountpoint.Root) + } + if mountpoint.Rw!= "/tmp/b" { + t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/b", mountpoint.Rw) + } +} + +func TestMountpointDuplicateRoot(t *testing.T) { + store, err := TempStore("test-mountpoint") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + _, err = image.Mountpoint("/tmp/a", "/tmp/b") + if err != nil { + t.Fatal(err) + } + if _, err = image.Mountpoint("/tmp/a", "/tmp/foobar"); err == nil { + t.Fatal("Duplicate mountpoint root should fail") + } +} + /* func TestMount(t *testing.T) { store, err := TempStore() From 6372a1a0d0b6506f25db7ba3dbba1e65fd6deb2d Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 18 Feb 2013 15:25:43 -0800 Subject: [PATCH 5/5] docker/fs: initial support for filesystem layers (adapted from image/layers.go) --- {image => fs}/layers.go | 16 +++++---- fs/layers_test.go | 80 +++++++++++++++++++++++++++++++++++++++++ fs/store.go | 14 +++++++- 3 files changed, 103 insertions(+), 7 deletions(-) rename {image => fs}/layers.go (86%) create mode 100644 fs/layers_test.go diff --git a/image/layers.go b/fs/layers.go similarity index 86% rename from image/layers.go rename to fs/layers.go index f856ff81d2..fa03ca5865 100644 --- a/image/layers.go +++ b/fs/layers.go @@ -1,4 +1,4 @@ -package image +package fs import ( "errors" @@ -20,6 +20,10 @@ func NewLayerStore(root string) (*LayerStore, error) { if err != nil { return nil, err } + // Create the root directory if it doesn't exists + if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + return nil, err + } return &LayerStore{ Root: abspath, }, nil @@ -82,7 +86,10 @@ func (store *LayerStore) layerPath(id string) string { } -func (store *LayerStore) AddLayer(archive io.Reader, stderr io.Writer, compression Compression) (string, error) { +func (store *LayerStore) AddLayer(id string, archive Archive, stderr io.Writer, compression Compression) (string, error) { + if _, err := os.Stat(store.layerPath(id)); err == nil { + return "", errors.New("Layer already exists: " + id) + } tmp, err := store.Mktemp() defer os.RemoveAll(tmp) if err != nil { @@ -110,14 +117,11 @@ func (store *LayerStore) AddLayer(archive io.Reader, stderr io.Writer, compressi } go io.Copy(stderr, untarStdout) untarCmd.Start() - hashR, hashW := io.Pipe() job_copy := future.Go(func() error { - _, err := io.Copy(io.MultiWriter(hashW, untarW), archive) - hashW.Close() + _, err := io.Copy(untarW, archive) untarW.Close() return err }) - id, err := future.ComputeId(hashR) if err != nil { return "", err } diff --git a/fs/layers_test.go b/fs/layers_test.go new file mode 100644 index 0000000000..3d8e9e32a7 --- /dev/null +++ b/fs/layers_test.go @@ -0,0 +1,80 @@ +package fs + +import ( + "io/ioutil" + "testing" + "os" + "github.com/dotcloud/docker/fake" +) + + + +func TestLayersInit(t *testing.T) { + store := tempStore(t) + defer os.RemoveAll(store.Root) + // Root should exist + if _, err := os.Stat(store.Root); err != nil { + t.Fatal(err) + } + // List() should be empty + if l := store.List(); len(l) != 0 { + t.Fatalf("List() should return %d, not %d", 0, len(l)) + } +} + +func TestAddLayer(t *testing.T) { + store := tempStore(t) + defer os.RemoveAll(store.Root) + layer, err := store.AddLayer("foo", testArchive(t), os.Stderr, Uncompressed) + if err != nil { + t.Fatal(err) + } + // Layer path should exist + if _, err := os.Stat(layer); err != nil { + t.Fatal(err) + } + // List() should return 1 layer + if l := store.List(); len(l) != 1 { + t.Fatalf("List() should return %d elements, not %d", 1, len(l)) + } + // Get("foo") should return the correct layer + if foo := store.Get("foo"); foo != layer { + t.Fatalf("get(\"foo\") should return '%d', not '%d'", layer, foo) + } +} + +func TestAddLayerDuplicate(t *testing.T) { + store := tempStore(t) + defer os.RemoveAll(store.Root) + if _, err := store.AddLayer("foobar123", testArchive(t), os.Stderr, Uncompressed); err != nil { + t.Fatal(err) + } + if _, err := store.AddLayer("foobar123", testArchive(t), os.Stderr, Uncompressed); err == nil { + t.Fatalf("Creating duplicate layer should fail") + } +} + + +/* + * HELPER FUNCTIONS + */ + +func tempStore(t *testing.T) *LayerStore { + tmp, err := ioutil.TempDir("", "docker-fs-layerstore-") + if err != nil { + t.Fatal(err) + } + store, err := NewLayerStore(tmp) + if err != nil { + t.Fatal(err) + } + return store +} + +func testArchive(t *testing.T) Archive { + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + return archive +} diff --git a/fs/store.go b/fs/store.go index 4ba23ae7dd..9bbc52e18e 100644 --- a/fs/store.go +++ b/fs/store.go @@ -14,6 +14,7 @@ type Store struct { Root string db *sql.DB orm *gorp.DbMap + layers *LayerStore } type Archive io.Reader @@ -33,10 +34,15 @@ func New(root string) (*Store, error) { if err := orm.CreateTables(); err != nil { return nil, err } + layers, err := NewLayerStore(path.Join(root, "layers")) + if err != nil { + return nil, err + } return &Store{ Root: root, db: db, orm: orm, + layers: layers, }, nil } @@ -88,13 +94,19 @@ func (store *Store) Get(id string) (*Image, error) { return img.(*Image), err } -func (store *Store) Create(layer Archive, parent *Image, pth, comment string) (*Image, error) { +func (store *Store) Create(layerData Archive, parent *Image, pth, comment string) (*Image, error) { // FIXME: actually do something with the layer... img := &Image{ Id : future.RandomId(), Comment: comment, store: store, } + // FIXME: we shouldn't have to pass os.Stderr to AddLayer()... + // FIXME: Archive should contain compression info. For now we only support uncompressed. + _, err := store.layers.AddLayer(img.Id, layerData, os.Stderr, Uncompressed) + if err != nil { + return nil, err + } path := &Path{ Path: path.Clean(pth), Image: img.Id,