From 44faa07b6c472b888bed65b56126cde56658e91a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 17:35:49 -0700 Subject: [PATCH 1/2] First integration of runtime with repositories & tags --- commands.go | 144 +++++++++++++++++++++++++----------------------- graph/graph.go | 13 +++++ graph/tags.go | 40 ++++++-------- runtime.go | 10 +++- runtime_test.go | 2 +- 5 files changed, 113 insertions(+), 96 deletions(-) diff --git a/commands.go b/commands.go index b127106ea9..e0d4fd0390 100644 --- a/commands.go +++ b/commands.go @@ -12,7 +12,6 @@ import ( "math/rand" "net/http" "net/url" - "path" "runtime" "strconv" "strings" @@ -356,32 +355,26 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "import", "[OPTIONS] NAME", "Create a new filesystem image from the contents of a tarball") - fl_stdin := cmd.Bool("stdin", false, "Read tarball from stdin") + cmd := rcli.Subcmd(stdout, "import", "[OPTIONS] URL|- [REPOSITORY [TAG]]", "Create a new filesystem image from the contents of a tarball") var archive io.Reader var resp *http.Response if err := cmd.Parse(args); err != nil { return nil } - name := cmd.Arg(0) - if name == "" { + src := cmd.Arg(0) + if src == "" { return errors.New("Not enough arguments") - } - if *fl_stdin { + } else if src == "-" { archive = stdin } else { - u, err := url.Parse(name) + u, err := url.Parse(src) if err != nil { return err } if u.Scheme == "" { u.Scheme = "http" } - if u.Host == "" { - u.Host = "get.docker.io" - u.Path = path.Join("/images", u.Path) - } fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() @@ -391,11 +384,17 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri } archive = ProgressReader(resp.Body, int(resp.ContentLength), stdout) } - fmt.Fprintf(stdout, "Unpacking to %s\n", name) - img, err := srv.runtime.graph.Create(archive, "", "") + img, err := srv.runtime.graph.Create(archive, "", "Imported from "+src) if err != nil { return err } + // Optionally register the image at REPO/TAG + if repository := cmd.Arg(1); repository != "" { + tag := cmd.Arg(2) // Repository will handle an empty tag properly + if err := srv.runtime.repositories.Set(repository, tag, img.Id); err != nil { + return err + } + } fmt.Fprintln(stdout, img.Id) return nil } @@ -411,68 +410,75 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd.Usage() return nil } - /* - var nameFilter string - if cmd.NArg() == 1 { - nameFilter = cmd.Arg(0) - } - */ + var nameFilter string + if cmd.NArg() == 1 { + nameFilter = cmd.Arg(0) + } w := tabwriter.NewWriter(stdout, 20, 1, 3, ' ', 0) if !*quiet { - fmt.Fprintf(w, "NAME\tID\tCREATED\tPARENT\n") + fmt.Fprintf(w, "REPOSITORY\tTAG\tID\tCREATED\tPARENT\n") } - if *quiet { - images, err := srv.runtime.graph.All() - if err != nil { - return err + allImages, err := srv.runtime.graph.Map() + if err != nil { + return err + } + for name, repository := range srv.runtime.repositories.Repositories { + if nameFilter != "" && name != nameFilter { + continue } - for _, image := range images { - fmt.Fprintln(stdout, image.Id) + for tag, id := range repository { + image, err := srv.runtime.graph.Get(id) + if err != nil { + log.Printf("Warning: couldn't load %s from %s/%s: %s", id, name, tag, err) + continue + } + delete(allImages, id) + if !*quiet { + for idx, field := range []string{ + /* REPOSITORY */ name, + /* TAG */ tag, + /* ID */ id, + /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", + /* PARENT */ image.Parent, + } { + if idx == 0 { + w.Write([]byte(field)) + } else { + w.Write([]byte("\t" + field)) + } + } + w.Write([]byte{'\n'}) + } else { + stdout.Write([]byte(image.Id + "\n")) + } } - } else { - // FIXME: - // paths, err := srv.images.Paths() - // if err != nil { - // return err - // } - // for _, name := range paths { - // if nameFilter != "" && nameFilter != name { - // continue - // } - // ids, err := srv.images.List(name) - // if err != nil { - // return err - // } - // for idx, img := range ids { - // if *limit > 0 && idx >= *limit { - // break - // } - // if !*quiet { - // for idx, field := range []string{ - // /* NAME */ name, - // /* ID */ img.Id, - // /* CREATED */ HumanDuration(time.Now().Sub(time.Unix(img.Created, 0))) + " ago", - // /* PARENT */ img.Parent, - // } { - // if idx == 0 { - // w.Write([]byte(field)) - // } else { - // w.Write([]byte("\t" + field)) - // } - // } - // w.Write([]byte{'\n'}) - // } else { - // stdout.Write([]byte(img.Id + "\n")) - // } - // } - // } - // if !*quiet { - // w.Flush() - // } - // + } + // Display images which aren't part of a + if nameFilter != "" { + for id, image := range allImages { + if !*quiet { + for idx, field := range []string{ + /* REPOSITORY */ "", + /* TAG */ "", + /* ID */ id, + /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", + /* PARENT */ image.Parent, + } { + if idx == 0 { + w.Write([]byte(field)) + } else { + w.Write([]byte("\t" + field)) + } + } + } else { + stdout.Write([]byte(image.Id + "\n")) + } + } + } + if !*quiet { + w.Flush() } return nil - } func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { diff --git a/graph/graph.go b/graph/graph.go index 7e0bc810cd..6470e17aea 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -122,6 +122,19 @@ func (graph *Graph) GarbageCollect() error { return os.RemoveAll(garbage.Root) } +func (graph *Graph) Map() (map[string]*Image, error) { + // FIXME: this should replace All() + all, err := graph.All() + if err != nil { + return nil, err + } + images := make(map[string]*Image, len(all)) + for _, image := range all { + images[image.Id] = image + } + return images, nil +} + func (graph *Graph) All() ([]*Image, error) { files, err := ioutil.ReadDir(graph.Root) if err != nil { diff --git a/graph/tags.go b/graph/tags.go index 337dc67e14..f3d702fcf7 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -6,33 +6,31 @@ import ( "path/filepath" ) -type RepoStore struct { +type TagStore struct { path string graph *Graph - Repositories map[string]*Repository + Repositories map[string]Repository } -type Repository struct { - Tags map[string]string -} +type Repository map[string]string -func NewRepoStore(path string, graph *Graph) (*RepoStore, error) { +func NewTagStore(path string, graph *Graph) (*TagStore, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err } - store := &RepoStore{ + store := &TagStore{ path: abspath, graph: graph, - Repositories: make(map[string]*Repository), + Repositories: make(map[string]Repository), } - if err := store.Reload(); err != nil { + if err := store.Save(); err != nil { return nil, err } return store, nil } -func (store *RepoStore) Save() error { +func (store *TagStore) Save() error { // Store the json ball jsonData, err := json.Marshal(store) if err != nil { @@ -44,7 +42,7 @@ func (store *RepoStore) Save() error { return nil } -func (store *RepoStore) Reload() error { +func (store *TagStore) Reload() error { jsonData, err := ioutil.ReadFile(store.path) if err != nil { return err @@ -55,22 +53,22 @@ func (store *RepoStore) Reload() error { return nil } -func (store *RepoStore) SetTag(repoName, tag, revision string) error { +func (store *TagStore) Set(repoName, tag, revision string) error { if err := store.Reload(); err != nil { return err } - var repo *Repository + var repo Repository if r, exists := store.Repositories[repoName]; exists { repo = r } else { - repo = NewRepository() + repo = make(map[string]string) store.Repositories[repoName] = repo } - repo.Tags[tag] = revision + repo[tag] = revision return store.Save() } -func (store *RepoStore) Get(repoName string) (*Repository, error) { +func (store *TagStore) Get(repoName string) (Repository, error) { if err := store.Reload(); err != nil { return nil, err } @@ -80,21 +78,15 @@ func (store *RepoStore) Get(repoName string) (*Repository, error) { return nil, nil } -func (store *RepoStore) GetImage(repoName, tag string) (*Image, error) { +func (store *TagStore) GetImage(repoName, tag string) (*Image, error) { repo, err := store.Get(repoName) if err != nil { return nil, err } else if repo == nil { return nil, nil } - if revision, exists := repo.Tags[tag]; exists { + if revision, exists := repo[tag]; exists { return store.graph.Get(revision) } return nil, nil } - -func NewRepository() *Repository { - return &Repository{ - Tags: make(map[string]string), - } -} diff --git a/runtime.go b/runtime.go index 8d017df690..ad543f0ed8 100644 --- a/runtime.go +++ b/runtime.go @@ -20,6 +20,7 @@ type Runtime struct { containers *list.List networkManager *NetworkManager graph *graph.Graph + repositories *graph.TagStore } var sysInitPath string @@ -201,10 +202,14 @@ func NewFromDirectory(root string) (*Runtime, error) { return nil, err } - graph, err := graph.New(path.Join(root, "graph")) + g, err := graph.New(path.Join(root, "graph")) if err != nil { return nil, err } + repositories, err := graph.NewTagStore(path.Join(root, "repositories"), g) + if err != nil { + return nil, fmt.Errorf("Couldn't create Tag store: %s", err) + } netManager, err := newNetworkManager(networkBridgeIface) if err != nil { return nil, err @@ -215,7 +220,8 @@ func NewFromDirectory(root string) (*Runtime, error) { repository: runtime_repo, containers: list.New(), networkManager: netManager, - graph: graph, + graph: g, + repositories: repositories, } if err := runtime.restore(); err != nil { diff --git a/runtime_test.go b/runtime_test.go index 093dae67fd..f97d049813 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -11,7 +11,7 @@ import ( ) const testLayerPath string = "/var/lib/docker/docker-ut.tar" -const unitTestImageName string = "busybox" +const unitTestImageName string = "http://get.docker.io/images/busybox" var unitTestStoreBase string var srv *Server From ef711962d5d70b4332908f0ca6460929e289c131 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 17:47:23 -0700 Subject: [PATCH 2/2] Folded graph/ back into main package --- graph/archive.go => archive.go | 2 +- graph/archive_test.go => archive_test.go | 2 +- graph/changes.go => changes.go | 2 +- commands.go | 2 +- container.go | 22 ++++++++-------------- graph/graph.go => graph.go | 8 ++++---- graph/graph_test.go => graph_test.go | 6 +++--- graph/image.go => image.go | 2 +- graph/mount.go => mount.go | 2 +- graph/mount_darwin.go => mount_darwin.go | 2 +- graph/mount_linux.go => mount_linux.go | 2 +- runtime.go | 15 +++++++-------- runtime_test.go | 13 ++++++------- graph/tags.go => tags.go | 2 +- 14 files changed, 37 insertions(+), 45 deletions(-) rename graph/archive.go => archive.go (99%) rename graph/archive_test.go => archive_test.go (98%) rename graph/changes.go => changes.go (99%) rename graph/graph.go => graph.go (95%) rename graph/graph_test.go => graph_test.go (98%) rename graph/image.go => image.go (99%) rename graph/mount.go => mount.go (98%) rename graph/mount_darwin.go => mount_darwin.go (92%) rename graph/mount_linux.go => mount_linux.go (92%) rename graph/tags.go => tags.go (99%) diff --git a/graph/archive.go b/archive.go similarity index 99% rename from graph/archive.go rename to archive.go index b6281bf4b5..78d4dfca0b 100644 --- a/graph/archive.go +++ b/archive.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "errors" diff --git a/graph/archive_test.go b/archive_test.go similarity index 98% rename from graph/archive_test.go rename to archive_test.go index c158537a6e..9f00aeccd7 100644 --- a/graph/archive_test.go +++ b/archive_test.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "io/ioutil" diff --git a/graph/changes.go b/changes.go similarity index 99% rename from graph/changes.go rename to changes.go index eebf7657e7..4c79918887 100644 --- a/graph/changes.go +++ b/changes.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "fmt" diff --git a/commands.go b/commands.go index e0d4fd0390..78d24ad0a0 100644 --- a/commands.go +++ b/commands.go @@ -803,7 +803,7 @@ func NewServer() (*Server, error) { // if err != nil { // return nil, err // } - runtime, err := New() + runtime, err := NewRuntime() if err != nil { return nil, err } diff --git a/container.go b/container.go index 2dcc3d0e28..ea091b8fa5 100644 --- a/container.go +++ b/container.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker/graph" "github.com/kr/pty" "io" "io/ioutil" @@ -63,11 +62,6 @@ type NetworkSettings struct { PortMapping map[string]string } -func GenerateId() string { - return graph.GenerateId() // Re-use the same code to generate container and image IDs - // (this might change when image Ids become content-based) -} - func (container *Container) Cmd() *exec.Cmd { return container.cmd } @@ -376,15 +370,15 @@ func (container *Container) Wait() int { return container.State.ExitCode } -func (container *Container) ExportRw() (graph.Archive, error) { - return graph.Tar(container.rwPath(), graph.Uncompressed) +func (container *Container) ExportRw() (Archive, error) { + return Tar(container.rwPath(), Uncompressed) } -func (container *Container) Export() (graph.Archive, error) { +func (container *Container) Export() (Archive, error) { if err := container.EnsureMounted(); err != nil { return nil, err } - return graph.Tar(container.RootfsPath(), graph.Uncompressed) + return Tar(container.RootfsPath(), Uncompressed) } func (container *Container) WaitTimeout(timeout time.Duration) error { @@ -420,7 +414,7 @@ func (container *Container) Mount() error { return image.Mount(container.RootfsPath(), container.rwPath()) } -func (container *Container) Changes() ([]graph.Change, error) { +func (container *Container) Changes() ([]Change, error) { image, err := container.GetImage() if err != nil { return nil, err @@ -428,7 +422,7 @@ func (container *Container) Changes() ([]graph.Change, error) { return image.Changes(container.rwPath()) } -func (container *Container) GetImage() (*graph.Image, error) { +func (container *Container) GetImage() (*Image, error) { if container.runtime == nil { return nil, fmt.Errorf("Can't get image of unregistered container") } @@ -436,11 +430,11 @@ func (container *Container) GetImage() (*graph.Image, error) { } func (container *Container) Mounted() (bool, error) { - return graph.Mounted(container.RootfsPath()) + return Mounted(container.RootfsPath()) } func (container *Container) Unmount() error { - return graph.Unmount(container.RootfsPath()) + return Unmount(container.RootfsPath()) } func (container *Container) logPath(name string) string { diff --git a/graph/graph.go b/graph.go similarity index 95% rename from graph/graph.go rename to graph.go index 6470e17aea..5001852c32 100644 --- a/graph/graph.go +++ b/graph.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "fmt" @@ -13,7 +13,7 @@ type Graph struct { Root string } -func New(root string) (*Graph, error) { +func NewGraph(root string) (*Graph, error) { abspath, err := filepath.Abs(root) if err != nil { return nil, err @@ -84,7 +84,7 @@ func (graph *Graph) Register(layerData Archive, img *Image) error { } func (graph *Graph) Mktemp(id string) (string, error) { - tmp, err := New(path.Join(graph.Root, ":tmp:")) + tmp, err := NewGraph(path.Join(graph.Root, ":tmp:")) if err != nil { return "", fmt.Errorf("Couldn't create temp: %s", err) } @@ -95,7 +95,7 @@ func (graph *Graph) Mktemp(id string) (string, error) { } func (graph *Graph) Garbage() (*Graph, error) { - return New(path.Join(graph.Root, ":garbage:")) + return NewGraph(path.Join(graph.Root, ":garbage:")) } func (graph *Graph) Delete(id string) error { diff --git a/graph/graph_test.go b/graph_test.go similarity index 98% rename from graph/graph_test.go rename to graph_test.go index d634ea85d5..20e879e52c 100644 --- a/graph/graph_test.go +++ b/graph_test.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "archive/tar" @@ -28,7 +28,7 @@ func TestInit(t *testing.T) { // FIXME: Do more extensive tests (ex: create multiple, delete, recreate; // create multiple, check the amount of images and paths, etc..) -func TestCreate(t *testing.T) { +func TestGraphCreate(t *testing.T) { graph := tempGraph(t) defer os.RemoveAll(graph.Root) archive, err := fakeTar() @@ -177,7 +177,7 @@ func tempGraph(t *testing.T) *Graph { if err != nil { t.Fatal(err) } - graph, err := New(tmp) + graph, err := NewGraph(tmp) if err != nil { t.Fatal(err) } diff --git a/graph/image.go b/image.go similarity index 99% rename from graph/image.go rename to image.go index 210ba7cedb..e508a164ea 100644 --- a/graph/image.go +++ b/image.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "bytes" diff --git a/graph/mount.go b/mount.go similarity index 98% rename from graph/mount.go rename to mount.go index ebf8485285..bb1a40eddb 100644 --- a/graph/mount.go +++ b/mount.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "fmt" diff --git a/graph/mount_darwin.go b/mount_darwin.go similarity index 92% rename from graph/mount_darwin.go rename to mount_darwin.go index 1fb24390b3..aeac78cda5 100644 --- a/graph/mount_darwin.go +++ b/mount_darwin.go @@ -1,4 +1,4 @@ -package graph +package docker import "errors" diff --git a/graph/mount_linux.go b/mount_linux.go similarity index 92% rename from graph/mount_linux.go rename to mount_linux.go index fd849821cf..0efb253003 100644 --- a/graph/mount_linux.go +++ b/mount_linux.go @@ -1,4 +1,4 @@ -package graph +package docker import "syscall" diff --git a/runtime.go b/runtime.go index ad543f0ed8..2c15c35937 100644 --- a/runtime.go +++ b/runtime.go @@ -3,7 +3,6 @@ package docker import ( "container/list" "fmt" - "github.com/dotcloud/docker/graph" "io" "io/ioutil" "log" @@ -19,8 +18,8 @@ type Runtime struct { repository string containers *list.List networkManager *NetworkManager - graph *graph.Graph - repositories *graph.TagStore + graph *Graph + repositories *TagStore } var sysInitPath string @@ -191,22 +190,22 @@ func (runtime *Runtime) restore() error { return nil } -func New() (*Runtime, error) { - return NewFromDirectory("/var/lib/docker") +func NewRuntime() (*Runtime, error) { + return NewRuntimeFromDirectory("/var/lib/docker") } -func NewFromDirectory(root string) (*Runtime, error) { +func NewRuntimeFromDirectory(root string) (*Runtime, error) { runtime_repo := path.Join(root, "containers") if err := os.MkdirAll(runtime_repo, 0700); err != nil && !os.IsExist(err) { return nil, err } - g, err := graph.New(path.Join(root, "graph")) + g, err := NewGraph(path.Join(root, "graph")) if err != nil { return nil, err } - repositories, err := graph.NewTagStore(path.Join(root, "repositories"), g) + repositories, err := NewTagStore(path.Join(root, "repositories"), g) if err != nil { return nil, fmt.Errorf("Couldn't create Tag store: %s", err) } diff --git a/runtime_test.go b/runtime_test.go index f97d049813..fe3ab4810f 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -1,7 +1,6 @@ package docker import ( - "github.com/dotcloud/docker/graph" "io" "io/ioutil" "os" @@ -57,7 +56,7 @@ func init() { unitTestStoreBase = root // Make it our Store root - runtime, err := NewFromDirectory(root) + runtime, err := NewRuntimeFromDirectory(root) if err != nil { panic(err) } @@ -84,7 +83,7 @@ func newTestRuntime() (*Runtime, error) { return nil, err } - runtime, err := NewFromDirectory(root) + runtime, err := NewRuntimeFromDirectory(root) if err != nil { return nil, err } @@ -92,7 +91,7 @@ func newTestRuntime() (*Runtime, error) { return runtime, nil } -func GetTestImage(runtime *Runtime) *graph.Image { +func GetTestImage(runtime *Runtime) *Image { imgs, err := runtime.graph.All() if err != nil { panic(err) @@ -102,7 +101,7 @@ func GetTestImage(runtime *Runtime) *graph.Image { return imgs[0] } -func TestCreate(t *testing.T) { +func TestRuntimeCreate(t *testing.T) { runtime, err := newTestRuntime() if err != nil { t.Fatal(err) @@ -269,7 +268,7 @@ func TestRestore(t *testing.T) { t.Fatal(err) } - runtime1, err := NewFromDirectory(root) + runtime1, err := NewRuntimeFromDirectory(root) if err != nil { t.Fatal(err) } @@ -294,7 +293,7 @@ func TestRestore(t *testing.T) { // Here are are simulating a docker restart - that is, reloading all containers // from scratch - runtime2, err := NewFromDirectory(root) + runtime2, err := NewRuntimeFromDirectory(root) if err != nil { t.Fatal(err) } diff --git a/graph/tags.go b/tags.go similarity index 99% rename from graph/tags.go rename to tags.go index f3d702fcf7..908691a8f9 100644 --- a/graph/tags.go +++ b/tags.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "encoding/json"