From 93ba6dd82bb62dbb9cac729cfa6626c469a87d00 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 8 Mar 2013 10:48:22 -0800 Subject: [PATCH] Plugged in image store in front-end (some commands are deactivated right now, will progressively reintroduce them --- container.go | 2 + dockerd/dockerd.go | 4 +- fs/changes.go | 144 +++++++++++++++++++++++ fs/store.go | 4 + server/server.go | 283 +++++++++++++++++++++++---------------------- 5 files changed, 295 insertions(+), 142 deletions(-) create mode 100644 fs/changes.go diff --git a/container.go b/container.go index ca7ab37f2d..3f727726d5 100644 --- a/container.go +++ b/container.go @@ -35,6 +35,7 @@ type Container struct { Config *Config Mountpoint *fs.Mountpoint State *State + Image string SysInitPath string lxcConfigPath string @@ -68,6 +69,7 @@ func createContainer(id string, root string, command string, args []string, imag Path: command, Args: args, Config: config, + Image: image.Id, Mountpoint: mountpoint, State: newState(), diff --git a/dockerd/dockerd.go b/dockerd/dockerd.go index b2337d342c..7fff1eb7dd 100644 --- a/dockerd/dockerd.go +++ b/dockerd/dockerd.go @@ -2,8 +2,8 @@ package main import ( "flag" - "github.com/dotcloud/docker" - "github.com/dotcloud/docker/server" + ".." + "../server" "log" ) diff --git a/fs/changes.go b/fs/changes.go new file mode 100644 index 0000000000..5c3523f3f3 --- /dev/null +++ b/fs/changes.go @@ -0,0 +1,144 @@ +package fs + +import ( + "fmt" + "path/filepath" + "os" + "strings" +) + +type ChangeType int + +const ( + ChangeModify = iota + ChangeAdd + ChangeDelete +) + +type Change struct { + Path string + Kind ChangeType +} + +func (change *Change) String() string { + var kind string + switch change.Kind { + case ChangeModify: + kind = "C" + case ChangeAdd: + kind = "A" + case ChangeDelete: + kind = "D" + } + return fmt.Sprintf("%s %s", kind, change.Path) +} + +func (store *Store) Changes(mp *Mountpoint) ([]Change, error) { + var changes []Change + image, err := store.Get(mp.Image) + if err != nil { + return nil, err + } + layers, err := image.layers() + if err != nil { + return nil, err + } + + err = filepath.Walk(mp.Rw, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(mp.Rw, path) + if err != nil { + return err + } + path = filepath.Join("/", path) + + // Skip root + if path == "/" { + return nil + } + + // Skip AUFS metadata + if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { + return err + } + + change := Change{ + Path: path, + } + + // Find out what kind of modification happened + file := filepath.Base(path) + // If there is a whiteout, then the file was removed + if strings.HasPrefix(file, ".wh.") { + originalFile := strings.TrimLeft(file, ".wh.") + change.Path = filepath.Join(filepath.Dir(path), originalFile) + change.Kind = ChangeDelete + } else { + // Otherwise, the file was added + change.Kind = ChangeAdd + + // ...Unless it already existed in a top layer, in which case, it's a modification + for _, layer := range layers { + stat, err := os.Stat(filepath.Join(layer, path)) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // The file existed in the top layer, so that's a modification + + // However, if it's a directory, maybe it wasn't actually modified. + // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar + if stat.IsDir() && f.IsDir() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { + // Both directories are the same, don't record the change + return nil + } + } + change.Kind = ChangeModify + break + } + } + } + + // Record change + changes = append(changes, change) + return nil + }) + if err != nil { + return nil, err + } + return changes, nil +} + +// Reset removes all changes to the filesystem, reverting it to its initial state. +func (mp *Mountpoint) Reset() error { + if err := os.RemoveAll(mp.Rw); err != nil { + return err + } + // We removed the RW directory itself along with its content: let's re-create an empty one. + if err := mp.createFolders(); err != nil { + return err + } + return nil +} + +// Open opens the named file for reading. +// func (fs *Filesystem) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { +// if err := fs.EnsureMounted(); err != nil { +// return nil, err +// } +// return os.OpenFile(filepath.Join(fs.RootFS, path), flag, perm) +// } + +// ReadDir reads the directory named by dirname, relative to the Filesystem's root, +// and returns a list of sorted directory entries +// func (fs *Filesystem) ReadDir(dirname string) ([]os.FileInfo, error) { +// if err := fs.EnsureMounted(); err != nil { +// return nil, err +// } +// return ioutil.ReadDir(filepath.Join(fs.RootFS, dirname)) +// } \ No newline at end of file diff --git a/fs/store.go b/fs/store.go index 9dd30e4507..67b0f2b3b4 100644 --- a/fs/store.go +++ b/fs/store.go @@ -157,6 +157,10 @@ func (store *Store) Register(image *Image, pth string) error { return trans.Commit() } +func (store *Store) Layers() []string { + return store.layers.List() +} + type Image struct { Id string Parent string diff --git a/server/server.go b/server/server.go index 8a3bb404f3..e5df5ec5ef 100644 --- a/server/server.go +++ b/server/server.go @@ -6,10 +6,10 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker" - "github.com/dotcloud/docker/future" - "github.com/dotcloud/docker/image" - "github.com/dotcloud/docker/rcli" + ".." + "../future" + "../fs" + "../rcli" "io" "net/http" "net/url" @@ -68,7 +68,7 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", len(srv.containers.List()), VERSION, - len(srv.images.ById)) + 0) // FIXME: Number of images return nil } @@ -153,7 +153,7 @@ func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...stri } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Filesystem.Umount(); err != nil { + if err := container.Mountpoint.Umount(); err != nil { return err } fmt.Fprintln(stdout, container.Id) @@ -176,7 +176,7 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Filesystem.Mount(); err != nil { + if err := container.Mountpoint.EnsureMounted(); err != nil { return err } fmt.Fprintln(stdout, container.Id) @@ -187,73 +187,73 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } -func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") - if err := cmd.Parse(args); err != nil { - cmd.Usage() - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if f, err := container.Filesystem.OpenFile(path, os.O_RDONLY, 0); err != nil { - return err - } else if _, err := io.Copy(stdout, f); err != nil { - return err - } - return nil - } - return errors.New("No such container: " + name) -} +// func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +// cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") +// if err := cmd.Parse(args); err != nil { +// cmd.Usage() +// return nil +// } +// if cmd.NArg() < 2 { +// cmd.Usage() +// return nil +// } +// name, path := cmd.Arg(0), cmd.Arg(1) +// if container := srv.containers.Get(name); container != nil { +// if f, err := container.Mountpoint.OpenFile(path, os.O_RDONLY, 0); err != nil { +// return err +// } else if _, err := io.Copy(stdout, f); err != nil { +// return err +// } +// return nil +// } +// return errors.New("No such container: " + name) +// } -func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") - if err := cmd.Parse(args); err != nil { - cmd.Usage() - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if f, err := container.Filesystem.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { - return err - } else if _, err := io.Copy(f, stdin); err != nil { - return err - } - return nil - } - return errors.New("No such container: " + name) -} +// func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +// cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") +// if err := cmd.Parse(args); err != nil { +// cmd.Usage() +// return nil +// } +// if cmd.NArg() < 2 { +// cmd.Usage() +// return nil +// } +// name, path := cmd.Arg(0), cmd.Arg(1) +// if container := srv.containers.Get(name); container != nil { +// if f, err := container.Mountpoint.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { +// return err +// } else if _, err := io.Copy(f, stdin); err != nil { +// return err +// } +// return nil +// } +// return errors.New("No such container: " + name) +// } -func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") - if err := cmd.Parse(args); err != nil { - cmd.Usage() - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if files, err := container.Filesystem.ReadDir(path); err != nil { - return err - } else { - for _, f := range files { - fmt.Fprintln(stdout, f.Name()) - } - } - return nil - } - return errors.New("No such container: " + name) -} +// func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +// cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") +// if err := cmd.Parse(args); err != nil { +// cmd.Usage() +// return nil +// } +// if cmd.NArg() < 2 { +// cmd.Usage() +// return nil +// } +// name, path := cmd.Arg(0), cmd.Arg(1) +// if container := srv.containers.Get(name); container != nil { +// if files, err := container.Mountpoint.ReadDir(path); err != nil { +// return err +// } else { +// for _, f := range files { +// fmt.Fprintln(stdout, f.Name()) +// } +// } +// return nil +// } +// return errors.New("No such container: " + name) +// } func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") @@ -269,8 +269,8 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str var obj interface{} if container := srv.containers.Get(name); container != nil { obj = container - } else if image := srv.images.Find(name); image != nil { - obj = image + //} else if image, err := srv.images.List(name); image != nil { + // obj = image } else { return errors.New("No such container or image: " + name) } @@ -289,27 +289,27 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str } // 'docker rmi NAME' removes all images with the name NAME -func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") - if err := cmd.Parse(args); err != nil { - cmd.Usage() - return nil - } - if cmd.NArg() < 1 { - cmd.Usage() - return nil - } - for _, name := range cmd.Args() { - image := srv.images.Find(name) - if image == nil { - return errors.New("No such image: " + name) - } - if err := srv.images.Delete(name); err != nil { - return err - } - } - return nil -} +// func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +// cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") +// if err := cmd.Parse(args); err != nil { +// cmd.Usage() +// return nil +// } +// if cmd.NArg() < 1 { +// cmd.Usage() +// return nil +// } +// for _, name := range cmd.Args() { +// image := srv.images.Find(name) +// if image == nil { +// return errors.New("No such image: " + name) +// } +// if err := srv.images.Delete(name); err != nil { +// return err +// } +// } +// return nil +// } func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "rm", "[OPTIONS] CONTAINER", "Remove a container") @@ -348,17 +348,9 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] NAME", "Download a new image from a remote location") - fl_bzip2 := cmd.Bool("j", false, "Bzip2 compression") - fl_gzip := cmd.Bool("z", false, "Gzip compression") if err := cmd.Parse(args); err != nil { return nil } - var compression image.Compression - if *fl_bzip2 { - compression = image.Bzip2 - } else if *fl_gzip { - compression = image.Gzip - } name := cmd.Arg(0) if name == "" { return errors.New("Not enough arguments") @@ -380,7 +372,7 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string if err != nil { return err } - img, err := srv.images.Import(name, resp.Body, stdout, nil, compression) + img, err := srv.images.Create(resp.Body, nil, name, "") if err != nil { return err } @@ -390,22 +382,14 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdPut(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "put", "[OPTIONS] NAME", "Import a new image from a local archive.") - fl_bzip2 := cmd.Bool("j", false, "Bzip2 compression") - fl_gzip := cmd.Bool("z", false, "Gzip compression") if err := cmd.Parse(args); err != nil { return nil } - var compression image.Compression - if *fl_bzip2 { - compression = image.Bzip2 - } else if *fl_gzip { - compression = image.Gzip - } name := cmd.Arg(0) if name == "" { return errors.New("Not enough arguments") } - img, err := srv.images.Import(name, stdin, stdout, nil, compression) + img, err := srv.images.Create(stdin, nil, name, "") if err != nil { return err } @@ -430,23 +414,27 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri if !*quiet { fmt.Fprintf(w, "NAME\tID\tCREATED\tPARENT\n") } - for _, name := range srv.images.Names() { + paths, err := srv.images.Paths() + if err != nil { + return err + } + for _, name := range paths { if nameFilter != "" && nameFilter != name { continue } - for idx, img := range *srv.images.ByName[name] { + ids, err := srv.images.List(name) + if err != nil { + return err + } + for idx, img := range ids { if *limit > 0 && idx >= *limit { break } if !*quiet { - id := img.Id - if !img.IdIsFinal() { - id += "..." - } for idx, field := range []string{ /* NAME */ name, - /* ID */ id, - /* CREATED */ future.HumanDuration(time.Now().Sub(img.Created)) + " ago", + /* ID */ img.Id, + /* CREATED */ future.HumanDuration(time.Now().Sub(time.Now())) + " ago", // FIXME: should be img.Created /* PARENT */ img.Parent, } { if idx == 0 { @@ -523,7 +511,7 @@ func (srv *Server) CmdLayers(stdin io.ReadCloser, stdout io.Writer, args ...stri if err := cmd.Parse(args); err != nil { return nil } - for _, layer := range srv.images.Layers.List() { + for _, layer := range srv.images.Layers() { fmt.Fprintln(stdout, layer) } return nil @@ -536,10 +524,16 @@ func (srv *Server) CmdCp(stdin io.ReadCloser, stdout io.Writer, args ...string) if err := cmd.Parse(args); err != nil { return nil } - if newImage, err := srv.images.Copy(cmd.Arg(0), cmd.Arg(1)); err != nil { + if image, err := srv.images.Get(cmd.Arg(0)); err != nil { return err + } else if image == nil { + return errors.New("Image " + cmd.Arg(0) + " does not exist") } else { - fmt.Fprintln(stdout, newImage.Id) + if img, err := image.Copy(cmd.Arg(1)); err != nil { + return err + } else { + fmt.Fprintln(stdout, img.Id) + } } return nil } @@ -558,16 +552,21 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri } if container := srv.containers.Get(containerName); container != nil { // FIXME: freeze the container before copying it to avoid data corruption? - rwTar, err := docker.Tar(container.Filesystem.RWPath) + rwTar, err := docker.Tar(container.Mountpoint.Rw) if err != nil { return err } // Create a new image from the container's base layers + a new layer from container changes - parentImg := srv.images.Find(container.GetUserData("image")) - img, err := srv.images.Import(imgName, rwTar, stdout, parentImg, image.Uncompressed) + parentImg, err := srv.images.Get(container.Image) if err != nil { return err } + + img, err := srv.images.Create(rwTar, parentImg, imgName, "") + if err != nil { + return err + } + fmt.Fprintln(stdout, img.Id) return nil } @@ -587,7 +586,10 @@ func (srv *Server) CmdTar(stdin io.ReadCloser, stdout io.Writer, args ...string) } name := cmd.Arg(0) if container := srv.containers.Get(name); container != nil { - data, err := container.Filesystem.Tar() + if err := container.Mountpoint.EnsureMounted(); err != nil { + return err + } + data, err := docker.Tar(container.Mountpoint.Root) if err != nil { return err } @@ -613,7 +615,7 @@ func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string if container := srv.containers.Get(cmd.Arg(0)); container == nil { return errors.New("No such container") } else { - changes, err := container.Filesystem.Changes() + changes, err := srv.images.Changes(container.Mountpoint) if err != nil { return err } @@ -636,7 +638,7 @@ func (srv *Server) CmdReset(stdin io.ReadCloser, stdout io.Writer, args ...strin } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Filesystem.Reset(); err != nil { + if err := container.Mountpoint.Reset(); err != nil { return errors.New("Reset " + container.Id + ": " + err.Error()) } } @@ -666,9 +668,9 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return errors.New("No such container: " + cmd.Arg(0)) } -func (srv *Server) CreateContainer(img *image.Image, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { +func (srv *Server) CreateContainer(img *fs.Image, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { id := future.RandomId()[:8] - container, err := srv.containers.Create(id, cmd, args, img.Layers, + container, err := srv.containers.Create(id, cmd, args, img, &docker.Config{Hostname: id, User: user, Tty: tty, OpenStdin: openStdin}) if err != nil { return nil, err @@ -757,8 +759,10 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) cmdline = []string{"/bin/bash", "-i"} } // Find the image - img := srv.images.Find(name) - if img == nil { + img, err := srv.images.Get(name) + if err != nil { + return err + } else if img == nil { return errors.New("No such image: " + name) } // Create new container @@ -820,16 +824,15 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) func New() (*Server, error) { future.Seed() - images, err := image.New("/var/lib/docker/images") - if err != nil { - return nil, err - } + // if err != nil { + // return nil, err + // } containers, err := docker.New() if err != nil { return nil, err } srv := &Server{ - images: images, + images: containers.Store, containers: containers, } return srv, nil @@ -876,5 +879,5 @@ func (srv *Server) CmdWeb(stdin io.ReadCloser, stdout io.Writer, args ...string) type Server struct { containers *docker.Docker - images *image.Store + images *fs.Store }