Merge branch 'fs' of github.com:dotcloud/docker into fs

fs/layers.go

@@ -1,4 +1,4 @@
-package image
+package fs
 
 import (
 	"errors"
@@ -20,6 +20,10 @@ func NewLayerStore(root string) (*LayerStore, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Create the root directory if it doesn't exists
+	if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) {
+		return nil, err
+	}
 	return &LayerStore{
 		Root: abspath,
 	}, nil
@@ -82,7 +86,10 @@ func (store *LayerStore) layerPath(id string) string {
 }
 
 
-func (store *LayerStore) AddLayer(archive io.Reader, stderr io.Writer, compression Compression) (string, error) {
+func (store *LayerStore) AddLayer(id string, archive Archive, stderr io.Writer, compression Compression) (string, error) {
+	if _, err := os.Stat(store.layerPath(id)); err == nil {
+		return "", errors.New("Layer already exists: " + id)
+	}
 	tmp, err := store.Mktemp()
 	defer os.RemoveAll(tmp)
 	if err != nil {
@@ -110,14 +117,11 @@ func (store *LayerStore) AddLayer(archive io.Reader, stderr io.Writer, compressi
 	}
 	go io.Copy(stderr, untarStdout)
 	untarCmd.Start()
-	hashR, hashW := io.Pipe()
 	job_copy := future.Go(func() error {
-		_, err := io.Copy(io.MultiWriter(hashW, untarW), archive)
+		_, err := io.Copy(untarW, archive)
-		hashW.Close()
 		untarW.Close()
 		return err
 	})
-	id, err := future.ComputeId(hashR)
 	if err != nil {
 		return "", err
 	}

fs/layers_test.go

@@ -0,0 +1,80 @@
+package fs
+
+import (
+	"io/ioutil"
+	"testing"
+	"os"
+	"github.com/dotcloud/docker/fake"
+)
+
+
+
+func TestLayersInit(t *testing.T) {
+	store := tempStore(t)
+	defer os.RemoveAll(store.Root)
+	// Root should exist
+	if _, err := os.Stat(store.Root); err != nil {
+		t.Fatal(err)
+	}
+	// List() should be empty
+	if l := store.List(); len(l) != 0 {
+		t.Fatalf("List() should return %d, not %d", 0, len(l))
+	}
+}
+
+func TestAddLayer(t *testing.T) {
+	store := tempStore(t)
+	defer os.RemoveAll(store.Root)
+	layer, err := store.AddLayer("foo", testArchive(t), os.Stderr, Uncompressed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Layer path should exist
+	if _, err := os.Stat(layer); err != nil {
+		t.Fatal(err)
+	}
+	// List() should return 1 layer
+	if l := store.List(); len(l) != 1 {
+		t.Fatalf("List() should return %d elements, not %d", 1, len(l))
+	}
+	// Get("foo") should return the correct layer
+	if foo := store.Get("foo"); foo != layer {
+		t.Fatalf("get(\"foo\") should return '%d', not '%d'", layer, foo)
+	}
+}
+
+func TestAddLayerDuplicate(t *testing.T) {
+	store := tempStore(t)
+	defer os.RemoveAll(store.Root)
+	if _, err := store.AddLayer("foobar123", testArchive(t), os.Stderr, Uncompressed); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := store.AddLayer("foobar123", testArchive(t), os.Stderr, Uncompressed); err == nil {
+		t.Fatalf("Creating duplicate layer should fail")
+	}
+}
+
+
+/*
+ * HELPER FUNCTIONS
+ */
+
+func tempStore(t *testing.T) *LayerStore {
+	tmp, err := ioutil.TempDir("", "docker-fs-layerstore-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	store, err := NewLayerStore(tmp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return store
+}
+
+func testArchive(t *testing.T) Archive {
+	archive, err := fake.FakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return archive
+}

fs/store.go

@@ -0,0 +1,190 @@
+package fs
+
+import (
+	"database/sql"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/coopernurse/gorp"
+	"os"
+	"io"
+	"path"
+	"github.com/dotcloud/docker/future"
+)
+
+type Store struct {
+	Root	string
+	db	*sql.DB
+	orm	*gorp.DbMap
+	layers	*LayerStore
+}
+
+type Archive io.Reader
+
+func New(root string) (*Store, error) {
+	if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+	db, err := sql.Open("sqlite3", path.Join(root, "db"))
+	if err != nil {
+		return nil, err
+	}
+	orm := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
+	orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id")
+	orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image")
+	orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root")
+	if err := orm.CreateTables(); err != nil {
+		return nil, err
+	}
+	layers, err := NewLayerStore(path.Join(root, "layers"))
+	if err != nil {
+		return nil, err
+	}
+	return &Store{
+		Root: root,
+		db: db,
+		orm: orm,
+		layers: layers,
+	}, nil
+}
+
+func (store *Store) imageList(src []interface{}) ([]*Image) {
+	var images []*Image
+	for _, i := range src {
+		img := i.(*Image)
+		img.store = store
+		images = append(images, img)
+	}
+	return images
+}
+
+func (store *Store) Images() ([]*Image, error) {
+	images , err := store.orm.Select(Image{}, "select * from images")
+	if err != nil {
+		return nil, err
+	}
+	return store.imageList(images), nil
+}
+
+func (store *Store) Paths() ([]string, error) {
+	var paths []string
+	rows, err := store.db.Query("select distinct Path from paths order by Path")
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		var path string
+		if err := rows.Scan(&path); err != nil {
+			return nil, err
+		}
+		paths = append(paths, path)
+	}
+	return paths, nil
+}
+
+func (store *Store) List(pth string) ([]*Image, error) {
+	pth = path.Clean(pth)
+	images, err := store.orm.Select(Image{}, "select images.* from images, paths where Path=? and paths.Image=images.Id", pth)
+	if err != nil {
+		return nil, err
+	}
+	return store.imageList(images), nil
+}
+
+func (store *Store) Get(id string) (*Image, error) {
+	img, err := store.orm.Get(Image{}, id)
+	return img.(*Image), err
+}
+
+func (store *Store) Create(layerData Archive, parent *Image, pth, comment string) (*Image, error) {
+	// FIXME: actually do something with the layer...
+	img := &Image{
+		Id :		future.RandomId(),
+		Comment:	comment,
+		store:		store,
+	}
+	// FIXME: we shouldn't have to pass os.Stderr to AddLayer()...
+	// FIXME: Archive should contain compression info. For now we only support uncompressed.
+	_, err := store.layers.AddLayer(img.Id, layerData, os.Stderr, Uncompressed)
+	if err != nil {
+		return nil, err
+	}
+	path := &Path{
+		Path:		path.Clean(pth),
+		Image:		img.Id,
+	}
+	trans, err := store.orm.Begin()
+	if err != nil {
+		return nil, err
+	}
+	if err := trans.Insert(img); err != nil {
+		return nil, err
+	}
+	if err := trans.Insert(path); err != nil {
+		return nil, err
+	}
+	if err := trans.Commit(); err != nil {
+		return nil, err
+	}
+	return img, nil
+}
+
+func (store *Store) Register(image *Image, pth string) error {
+	image.store = store
+	// FIXME: import layer
+	trans, err := store.orm.Begin()
+	if err != nil {
+		return err
+	}
+	trans.Insert(image)
+	trans.Insert(&Path{Path: pth, Image: image.Id})
+	return trans.Commit()
+}
+
+
+
+
+type Image struct {
+	Id		string
+	Parent		string
+	Comment		string
+	store		*Store	`db:"-"`
+}
+
+
+func (image *Image) Copy(pth string) (*Image, error) {
+	if err := image.store.orm.Insert(&Path{Path: pth, Image: image.Id}); err != nil {
+		return nil, err
+	}
+	return image, nil
+}
+
+type Mountpoint struct {
+	Image	string
+	Root	string
+	Rw	string
+}
+
+func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) {
+	mountpoint := &Mountpoint{Root: path.Clean(root), Rw: path.Clean(rw), Image: image.Id}
+	if err := image.store.orm.Insert(mountpoint); err != nil {
+		return nil, err
+	}
+	return mountpoint, nil
+}
+
+func (image *Image) Mountpoints() ([]*Mountpoint, error) {
+	var mountpoints []*Mountpoint
+	res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=?", image.Id)
+	if err != nil {
+		return nil, err
+	}
+	for _, mp := range res {
+		mountpoints = append(mountpoints, mp.(*Mountpoint))
+	}
+	return mountpoints, nil
+}
+
+
+type Path struct {
+	Path	string
+	Image	string
+}

fs/store_test.go

@@ -0,0 +1,266 @@
+package fs
+
+import (
+	"testing"
+	"io/ioutil"
+	"github.com/dotcloud/docker/fake"
+	"os"
+	"errors"
+	"fmt"
+)
+
+func TestInit(t *testing.T) {
+	store, err := TempStore("testinit")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(store)
+	paths, err := store.Paths()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if l := len(paths); l != 0 {
+		t.Fatal("Fresh store should be empty after init (len=%d)", l)
+	}
+}
+
+func TestCreate(t *testing.T) {
+	store, err := TempStore("testcreate")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(store)
+	archive, err := fake.FakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	image, err := store.Create(archive, nil, "foo", "Testing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if images, err := store.Images(); err != nil {
+		t.Fatal(err)
+	} else if l := len(images); l != 1 {
+		t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l)
+	}
+	if images, err := store.List("foo"); err != nil {
+		t.Fatal(err)
+	} else if l := len(images); l != 1 {
+		t.Fatalf("Path foo has wrong number of images (should be %d, not %d)", 1, l)
+	} else if images[0].Id != image.Id {
+		t.Fatalf("Imported image should be listed at path foo (%s != %s)", images[0], image)
+	}
+}
+
+// Copy an image to a new path
+func TestCopyNewPath(t *testing.T) {
+	store, err := TempStore("testcopynewpath")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(store)
+	archive, err := fake.FakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	src, err := store.Create(archive, nil, "foo", "Testing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	dst, err := src.Copy("bar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// ID should be the same
+	if src.Id != dst.Id {
+		t.Fatal("Different IDs")
+	}
+	// Check number of images at source path
+	if images, err := store.List("foo"); err != nil {
+		t.Fatal(err)
+	} else if l := len(images); l != 1 {
+		t.Fatal("Wrong number of images at source path (should be %d, not %d)", 1, l)
+	}
+	// Check number of images at destination path
+	if images, err := store.List("bar"); err != nil {
+		t.Fatal(err)
+	} else if l := len(images); l != 1 {
+		t.Fatal("Wrong number of images at destination path (should be %d, not %d)", 1, l)
+	}
+	if err := healthCheck(store); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Copying an image to the same path twice should fail
+func TestCopySameName(t *testing.T) {
+	store, err := TempStore("testcopysamename")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(store)
+	archive, err := fake.FakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	src, err := store.Create(archive, nil, "foo", "Testing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = src.Copy("foo")
+	if err == nil {
+		t.Fatal("Copying an image to the same patch twice should fail.")
+	}
+}
+
+func TestMountPoint(t *testing.T) {
+	store, err := TempStore("test-mountpoint")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(store)
+	archive, err := fake.FakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	image, err := store.Create(archive, nil, "foo", "Testing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	mountpoint, err := image.Mountpoint("/tmp/a", "/tmp/b")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if mountpoint.Root != "/tmp/a" {
+		t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/a", mountpoint.Root)
+	}
+	if mountpoint.Rw!= "/tmp/b" {
+		t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/b", mountpoint.Rw)
+	}
+}
+
+func TestMountpointDuplicateRoot(t *testing.T) {
+	store, err := TempStore("test-mountpoint")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(store)
+	archive, err := fake.FakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	image, err := store.Create(archive, nil, "foo", "Testing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = image.Mountpoint("/tmp/a", "/tmp/b")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err = image.Mountpoint("/tmp/a", "/tmp/foobar"); err == nil {
+		t.Fatal("Duplicate mountpoint root should fail")
+	}
+}
+
+/*
+func TestMount(t *testing.T) {
+	store, err := TempStore()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(store)
+	archive, err := fake.FakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	image, err := store.Create(archive, nil, "foo", "Testing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Create mount targets
+	root, err := ioutil.TempDir("", "docker-fs-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	rw, err := ioutil.TempDir("", "docker-fs-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	mountpoint, err := image.Mount(root, rw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer mountpoint.Umount()
+	// Mountpoint should be marked as mounted
+	if !mountpoint.Mounted() {
+		t.Fatal("Mountpoint not mounted")
+	}
+	// There should be one mountpoint registered
+	if l := len(image.Mountpoints()); l != 1 {
+		t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 1, l)
+	}
+	// Unmounting should work
+	if err := mountpoint.Umount(); err != nil {
+		t.Fatal(err)
+	}
+	// De-registering should work
+	if err := mountpoint.Deregister(); err != nil {
+		t.Fatal(err)
+	}
+	if l := len(image.Mountpoints()); l != 0 {
+		t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 0, l)
+	}
+	// General health check
+	if err := healthCheck(); err != nil {
+		t.Fatal(err)
+	}
+}
+*/
+
+func TempStore(prefix string) (*Store, error) {
+	dir, err := ioutil.TempDir("", "docker-fs-test-" + prefix)
+	if err != nil {
+		return nil, err
+	}
+	return New(dir)
+}
+
+func nuke(store *Store) error {
+	return os.RemoveAll(store.Root)
+}
+
+// Look for inconsistencies in a store.
+func healthCheck(store *Store) error {
+	parents := make(map[string]bool)
+	paths, err := store.Paths()
+	if err != nil {
+		return err
+	}
+	for _, path := range paths {
+		images, err := store.List(path)
+		if err != nil {
+			return err
+		}
+		IDs := make(map[string]bool) // All IDs for this path
+		for _, img := range images {
+			// Check for duplicate IDs per path
+			if _, exists := IDs[img.Id]; exists {
+				return errors.New(fmt.Sprintf("Duplicate ID: %s", img.Id))
+			} else {
+				IDs[img.Id] = true
+			}
+			// Store parent for 2nd pass
+			if parent := img.Parent; parent != "" {
+				parents[parent] = true
+			}
+		}
+	}
+	// Check non-existing parents
+	for parent := range parents {
+		if _, exists := parents[parent]; !exists {
+			return errors.New("Reference to non-registered parent: " + parent)
+		}
+	}
+	return nil
+}
+