diff --git a/.gitignore b/.gitignore index 94f63b9f26..cabd399067 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vagrant +bin docker/docker .*.swp a.out @@ -6,3 +7,5 @@ a.out build_src command-line-arguments.test .flymake* +docker.test +auth/auth.test diff --git a/README.md b/README.md index 0a9b69c6de..851ddb4fce 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Installing on Ubuntu 12.04 and 12.10 ```bash cd docker-master - sudo ./docker run -a -i -t base /bin/bash + sudo ./docker run -i -t base /bin/bash ``` Consider adding docker to your `PATH` for simplicity. @@ -136,7 +136,7 @@ docker import base # Run an interactive shell in the base image, # allocate a tty, attach stdin and stdout -docker run -a -i -t base /bin/bash +docker run -i -t base /bin/bash ``` @@ -148,7 +148,7 @@ Starting a long-running worker process (docker -d || echo "Docker daemon already running") & # Start a very useful long-running process -JOB=$(docker run base /bin/sh -c "while true; do echo Hello world; sleep 1; done") +JOB=$(docker run -d base /bin/sh -c "while true; do echo Hello world; sleep 1; done") # Collect the output of the job so far docker logs $JOB @@ -171,7 +171,7 @@ Expose a service on a TCP port ```bash # Expose port 4444 of this container, and tell netcat to listen on it -JOB=$(docker run -p 4444 base /bin/nc -l -p 4444) +JOB=$(docker run -d -p 4444 base /bin/nc -l -p 4444) # Which public port is NATed to my container? PORT=$(docker port $JOB 4444) @@ -227,7 +227,7 @@ Setting up a dev environment Instructions that have been verified to work on Ubuntu 12.10, ```bash -sudo apt-get -y install lxc wget bsdtar curl libsqlite3-dev golang git pkg-config +sudo apt-get -y install lxc wget bsdtar curl golang git export GOPATH=~/go/ export PATH=$GOPATH/bin:$PATH diff --git a/fs/archive.go b/archive.go similarity index 97% rename from fs/archive.go rename to archive.go index f43cc64b66..78d4dfca0b 100644 --- a/fs/archive.go +++ b/archive.go @@ -1,4 +1,4 @@ -package fs +package docker import ( "errors" @@ -7,6 +7,8 @@ import ( "os/exec" ) +type Archive io.Reader + type Compression uint32 const ( diff --git a/fs/archive_test.go b/archive_test.go similarity index 98% rename from fs/archive_test.go rename to archive_test.go index b182a1563e..9f00aeccd7 100644 --- a/fs/archive_test.go +++ b/archive_test.go @@ -1,4 +1,4 @@ -package fs +package docker import ( "io/ioutil" diff --git a/auth/auth.go b/auth/auth.go index 498b7d8413..bbc9e32640 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -8,11 +8,12 @@ import ( "io/ioutil" "net/http" "os" + "path" "strings" ) // Where we store the config file -const CONFIGFILE = "/var/lib/docker/.dockercfg" +const CONFIGFILE = ".dockercfg" // the registry server we want to login against const REGISTRY_SERVER = "https://registry.docker.io" @@ -21,10 +22,20 @@ type AuthConfig struct { Username string `json:"username"` Password string `json:"password"` Email string `json:"email"` + rootPath string `json:-` +} + +func NewAuthConfig(username, password, email, rootPath string) *AuthConfig { + return &AuthConfig{ + Username: username, + Password: password, + Email: email, + rootPath: rootPath, + } } // create a base64 encoded auth string to store in config -func EncodeAuth(authConfig AuthConfig) string { +func EncodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password msg := []byte(authStr) encoded := make([]byte, base64.StdEncoding.EncodedLen(len(msg))) @@ -33,50 +44,54 @@ func EncodeAuth(authConfig AuthConfig) string { } // decode the auth string -func DecodeAuth(authStr string) (AuthConfig, error) { +func DecodeAuth(authStr string) (*AuthConfig, error) { decLen := base64.StdEncoding.DecodedLen(len(authStr)) decoded := make([]byte, decLen) authByte := []byte(authStr) n, err := base64.StdEncoding.Decode(decoded, authByte) if err != nil { - return AuthConfig{}, err + return nil, err } if n > decLen { - return AuthConfig{}, errors.New("something went wrong decoding auth config") + return nil, fmt.Errorf("Something went wrong decoding auth config") } arr := strings.Split(string(decoded), ":") + if len(arr) != 2 { + return nil, fmt.Errorf("Invalid auth configuration file") + } password := strings.Trim(arr[1], "\x00") - return AuthConfig{Username: arr[0], Password: password}, nil + return &AuthConfig{Username: arr[0], Password: password}, nil } // load up the auth config information and return values -func LoadConfig() (AuthConfig, error) { - if _, err := os.Stat(CONFIGFILE); err == nil { - b, err := ioutil.ReadFile(CONFIGFILE) - if err != nil { - return AuthConfig{}, err - } - arr := strings.Split(string(b), "\n") - orig_auth := strings.Split(arr[0], " = ") - orig_email := strings.Split(arr[1], " = ") - authConfig, err := DecodeAuth(orig_auth[1]) - if err != nil { - return AuthConfig{}, err - } - authConfig.Email = orig_email[1] - return authConfig, nil - } else { - return AuthConfig{}, nil +// FIXME: use the internal golang config parser +func LoadConfig(rootPath string) (*AuthConfig, error) { + confFile := path.Join(rootPath, CONFIGFILE) + if _, err := os.Stat(confFile); err != nil { + return &AuthConfig{}, fmt.Errorf("The Auth config file is missing") } - return AuthConfig{}, nil + b, err := ioutil.ReadFile(confFile) + if err != nil { + return nil, err + } + arr := strings.Split(string(b), "\n") + orig_auth := strings.Split(arr[0], " = ") + orig_email := strings.Split(arr[1], " = ") + authConfig, err := DecodeAuth(orig_auth[1]) + if err != nil { + return nil, err + } + authConfig.Email = orig_email[1] + authConfig.rootPath = rootPath + return authConfig, nil } // save the auth config -func saveConfig(authStr string, email string) error { +func saveConfig(rootPath, authStr string, email string) error { lines := "auth = " + authStr + "\n" + "email = " + email + "\n" b := []byte(lines) - err := ioutil.WriteFile(CONFIGFILE, b, 0600) + err := ioutil.WriteFile(path.Join(rootPath, CONFIGFILE), b, 0600) if err != nil { return err } @@ -84,7 +99,7 @@ func saveConfig(authStr string, email string) error { } // try to register/login to the registry server -func Login(authConfig AuthConfig) (string, error) { +func Login(authConfig *AuthConfig) (string, error) { storeConfig := false reqStatusCode := 0 var status string @@ -145,7 +160,7 @@ func Login(authConfig AuthConfig) (string, error) { } if storeConfig { authStr := EncodeAuth(authConfig) - saveConfig(authStr, authConfig.Email) + saveConfig(authConfig.rootPath, authStr, authConfig.Email) } return status, nil } diff --git a/auth/auth_test.go b/auth/auth_test.go index d1650668e7..ca584f9314 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -5,7 +5,7 @@ import ( ) func TestEncodeAuth(t *testing.T) { - newAuthConfig := AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} + newAuthConfig := &AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} authStr := EncodeAuth(newAuthConfig) decAuthConfig, err := DecodeAuth(authStr) if err != nil { diff --git a/fs/changes.go b/changes.go similarity index 75% rename from fs/changes.go rename to changes.go index 659f688c45..4c79918887 100644 --- a/fs/changes.go +++ b/changes.go @@ -1,4 +1,4 @@ -package fs +package docker import ( "fmt" @@ -33,24 +33,15 @@ func (change *Change) String() string { return fmt.Sprintf("%s %s", kind, change.Path) } -func (store *Store) Changes(mp *Mountpoint) ([]Change, error) { +func Changes(layers []string, rw string) ([]Change, error) { var changes []Change - image, err := store.Get(mp.Image) - if err != nil { - return nil, err - } - layers, err := image.layers() - if err != nil { - return nil, err - } - - err = filepath.Walk(mp.Rw, func(path string, f os.FileInfo, err error) error { + err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error { if err != nil { return err } // Rebase path - path, err = filepath.Rel(mp.Rw, path) + path, err = filepath.Rel(rw, path) if err != nil { return err } @@ -113,15 +104,3 @@ func (store *Store) Changes(mp *Mountpoint) ([]Change, error) { } return changes, nil } - -// Reset removes all changes to the filesystem, reverting it to its initial state. -func (mp *Mountpoint) Reset() error { - if err := os.RemoveAll(mp.Rw); err != nil { - return err - } - // We removed the RW directory itself along with its content: let's re-create an empty one. - if err := mp.createFolders(); err != nil { - return err - } - return nil -} diff --git a/commands.go b/commands.go index 072c3dde4e..92232accc2 100644 --- a/commands.go +++ b/commands.go @@ -1,22 +1,17 @@ package docker import ( - "bufio" "bytes" "encoding/json" "errors" "fmt" "github.com/dotcloud/docker/auth" - "github.com/dotcloud/docker/fs" - "github.com/dotcloud/docker/future" "github.com/dotcloud/docker/rcli" "io" - "io/ioutil" "log" + "math/rand" "net/http" "net/url" - "os" - "path" "runtime" "strconv" "strings" @@ -39,37 +34,30 @@ func (srv *Server) Help() string { {"ps", "Display a list of containers"}, {"import", "Create a new filesystem image from the contents of a tarball"}, {"attach", "Attach to a running container"}, - {"cat", "Write the contents of a container's file to standard output"}, {"commit", "Create a new image from a container's changes"}, - {"cp", "Create a copy of IMAGE and call it NAME"}, - {"debug", "(debug only) (No documentation available)"}, + {"history", "Show the history of an image"}, {"diff", "Inspect changes on a container's filesystem"}, {"images", "List images"}, {"info", "Display system-wide information"}, {"inspect", "Return low-level information on a container"}, {"kill", "Kill a running container"}, - {"layers", "(debug only) List filesystem layers"}, {"login", "Register or Login to the docker registry server"}, {"logs", "Fetch the logs of a container"}, - {"ls", "List the contents of a container's directory"}, - {"mirror", "(debug only) (No documentation available)"}, {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, {"ps", "List containers"}, - {"reset", "Reset changes to a container's filesystem"}, + {"pull", "Pull an image or a repository to the docker registry server"}, + {"push", "Push an image or a repository to the docker registry server"}, {"restart", "Restart a running container"}, {"rm", "Remove a container"}, {"rmi", "Remove an image"}, {"run", "Run a command in a new container"}, {"start", "Start a stopped container"}, {"stop", "Stop a running container"}, - {"tar", "Stream the contents of a container as a tar archive"}, - {"umount", "(debug only) Mount a container's filesystem"}, + {"export", "Stream the contents of a container as a tar archive"}, {"version", "Show the docker version information"}, {"wait", "Block until a container stops, then print its exit code"}, - {"web", "A web UI for docker"}, - {"write", "Write the contents of standard input to a container's file"}, } { - help += fmt.Sprintf(" %-10.10s%s\n", cmd...) + help += fmt.Sprintf(" %-10.10s%s\n", cmd[0], cmd[1]) } return help } @@ -83,17 +71,13 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout io.Writer, args ...strin var username string var password string var email string - authConfig, err := auth.LoadConfig() - if err != nil { - fmt.Fprintf(stdout, "Error : %s\n", err) - } - fmt.Fprint(stdout, "Username (", authConfig.Username, "): ") + fmt.Fprint(stdout, "Username (", srv.runtime.authConfig.Username, "): ") fmt.Fscanf(stdin, "%s", &username) if username == "" { - username = authConfig.Username + username = srv.runtime.authConfig.Username } - if username != authConfig.Username { + if username != srv.runtime.authConfig.Username { fmt.Fprint(stdout, "Password: ") fmt.Fscanf(stdin, "%s", &password) @@ -101,16 +85,16 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout io.Writer, args ...strin return errors.New("Error : Password Required\n") } - fmt.Fprint(stdout, "Email (", authConfig.Email, "): ") + fmt.Fprint(stdout, "Email (", srv.runtime.authConfig.Email, "): ") fmt.Fscanf(stdin, "%s", &email) if email == "" { - email = authConfig.Email + email = srv.runtime.authConfig.Email } } else { - password = authConfig.Password - email = authConfig.Email + password = srv.runtime.authConfig.Password + email = srv.runtime.authConfig.Email } - newAuthConfig := auth.AuthConfig{Username: username, Password: password, Email: email} + newAuthConfig := auth.NewAuthConfig(username, password, email, srv.runtime.root) status, err := auth.Login(newAuthConfig) if err != nil { fmt.Fprintf(stdout, "Error : %s\n", err) @@ -132,7 +116,7 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { fmt.Fprintln(stdout, container.Wait()) } else { return errors.New("No such container: " + name) @@ -149,7 +133,7 @@ func (srv *Server) CmdVersion(stdin io.ReadCloser, stdout io.Writer, args ...str // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - images, _ := srv.images.Images() + images, _ := srv.runtime.graph.All() var imgcount int if images == nil { imgcount = 0 @@ -165,7 +149,7 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", - len(srv.containers.List()), + len(srv.runtime.List()), VERSION, imgcount) return nil @@ -181,7 +165,7 @@ func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { if err := container.Stop(); err != nil { return err } @@ -203,7 +187,7 @@ func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...str return nil } for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { if err := container.Restart(); err != nil { return err } @@ -225,7 +209,7 @@ func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { if err := container.Start(); err != nil { return err } @@ -237,115 +221,6 @@ func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } -func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "umount a container's filesystem (debug only)") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 1 { - cmd.Usage() - return nil - } - for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { - if err := container.Mountpoint.Umount(); err != nil { - return err - } - fmt.Fprintln(stdout, container.Id) - } else { - return errors.New("No such container: " + name) - } - } - return nil -} - -func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "mount a container's filesystem (debug only)") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 1 { - cmd.Usage() - return nil - } - for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { - if err := container.Mountpoint.EnsureMounted(); err != nil { - return err - } - fmt.Fprintln(stdout, container.Id) - } else { - return errors.New("No such container: " + name) - } - } - return nil -} - -func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if f, err := container.Mountpoint.OpenFile(path, os.O_RDONLY, 0); err != nil { - return err - } else if _, err := io.Copy(stdout, f); err != nil { - return err - } - return nil - } - return errors.New("No such container: " + name) -} - -func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if f, err := container.Mountpoint.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { - return err - } else if _, err := io.Copy(f, stdin); err != nil { - return err - } - return nil - } - return errors.New("No such container: " + name) -} - -func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if files, err := container.Mountpoint.ReadDir(path); err != nil { - return err - } else { - for _, f := range files { - fmt.Fprintln(stdout, f.Name()) - } - } - return nil - } - return errors.New("No such container: " + name) -} - func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") if err := cmd.Parse(args); err != nil { @@ -357,11 +232,9 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str } name := cmd.Arg(0) var obj interface{} - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { obj = container - } else if image, err := srv.images.Find(name); err != nil { - return err - } else if image != nil { + } else if image, err := srv.runtime.repositories.LookupImage(name); err == nil && image != nil { obj = image } else { // No output means the object does not exist @@ -394,7 +267,7 @@ func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string } name := cmd.Arg(0) privatePort := cmd.Arg(1) - if container := srv.containers.Get(name); container == nil { + if container := srv.runtime.Get(name); container == nil { return errors.New("No such container: " + name) } else { if frontend, exists := container.NetworkSettings.PortMapping[privatePort]; !exists { @@ -409,44 +282,52 @@ func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string // 'docker rmi NAME' removes all images with the name NAME func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) (err error) { cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") - fl_all := cmd.Bool("a", false, "Use IMAGE as a path and remove ALL images in this path") - fl_regexp := cmd.Bool("r", false, "Use IMAGE as a regular expression instead of an exact name") if cmd.Parse(args) != nil || cmd.NArg() < 1 { cmd.Usage() return nil } for _, name := range cmd.Args() { - if *fl_regexp { - err = srv.images.RemoveRegexp(name) - } else if *fl_all { - err = srv.images.RemoveInPath(name) - } else { - if image, err1 := srv.images.Find(name); err1 != nil { - err = err1 - } else if err1 == nil && image == nil { - err = fmt.Errorf("No such image: %s", name) - } else { - err = srv.images.Remove(image) - } - } - if err != nil { + if err := srv.runtime.graph.Delete(name); err != nil { return err } } return nil } +func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "history", "[OPTIONS] IMAGE", "Show the history of an image") + if cmd.Parse(args) != nil || cmd.NArg() != 1 { + cmd.Usage() + return nil + } + image, err := srv.runtime.repositories.LookupImage(cmd.Arg(0)) + if err != nil { + return err + } + w := tabwriter.NewWriter(stdout, 20, 1, 3, ' ', 0) + defer w.Flush() + fmt.Fprintf(w, "ID\tCREATED\tCREATED BY\n") + return image.WalkHistory(func(img *Image) error { + fmt.Fprintf(w, "%s\t%s\t%s\n", + srv.runtime.repositories.ImageName(img.Id), + HumanDuration(time.Now().Sub(img.Created))+" ago", + strings.Join(img.ContainerConfig.Cmd, " "), + ) + return nil + }) +} + func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "rm", "[OPTIONS] CONTAINER", "Remove a container") if err := cmd.Parse(args); err != nil { return nil } for _, name := range cmd.Args() { - container := srv.containers.Get(name) + container := srv.runtime.Get(name) if container == nil { return errors.New("No such container: " + name) } - if err := srv.containers.Destroy(container); err != nil { + if err := srv.runtime.Destroy(container); err != nil { fmt.Fprintln(stdout, "Error destroying container "+name+": "+err.Error()) } } @@ -460,7 +341,7 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } for _, name := range cmd.Args() { - container := srv.containers.Get(name) + container := srv.runtime.Get(name) if container == nil { return errors.New("No such container: " + name) } @@ -472,54 +353,147 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "import", "[OPTIONS] NAME", "Create a new filesystem image from the contents of a tarball") - fl_stdin := cmd.Bool("stdin", false, "Read tarball from stdin") + cmd := rcli.Subcmd(stdout, "import", "[OPTIONS] URL|- [REPOSITORY [TAG]]", "Create a new filesystem image from the contents of a tarball") var archive io.Reader var resp *http.Response if err := cmd.Parse(args); err != nil { return nil } - name := cmd.Arg(0) - if name == "" { + src := cmd.Arg(0) + if src == "" { return errors.New("Not enough arguments") - } - if *fl_stdin { + } else if src == "-" { archive = stdin } else { - u, err := url.Parse(name) + u, err := url.Parse(src) if err != nil { return err } if u.Scheme == "" { u.Scheme = "http" - } - if u.Host == "" { - u.Host = "get.docker.io" - u.Path = path.Join("/images", u.Path) + u.Host = src + u.Path = "" } fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() - resp, err = future.Download(u.String(), stdout) + resp, err = Download(u.String(), stdout) if err != nil { return err } - archive = future.ProgressReader(resp.Body, int(resp.ContentLength), stdout) + archive = ProgressReader(resp.Body, int(resp.ContentLength), stdout) } - fmt.Fprintf(stdout, "Unpacking to %s\n", name) - img, err := srv.images.Create(archive, nil, name, "") + img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src) if err != nil { return err } + // Optionally register the image at REPO/TAG + if repository := cmd.Arg(1); repository != "" { + tag := cmd.Arg(2) // Repository will handle an empty tag properly + if err := srv.runtime.repositories.Set(repository, tag, img.Id, true); err != nil { + return err + } + } fmt.Fprintln(stdout, img.Id) return nil } +func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "push", "LOCAL", "Push an image or a repository to the registry") + if err := cmd.Parse(args); err != nil { + return nil + } + local := cmd.Arg(0) + + if local == "" { + cmd.Usage() + return nil + } + + // If the login failed, abort + if srv.runtime.authConfig == nil { + return fmt.Errorf("Please login prior to push. ('docker login')") + } + + var remote string + + tmp := strings.SplitN(local, "/", 2) + if len(tmp) == 1 { + remote = srv.runtime.authConfig.Username + "/" + local + } else { + remote = local + } + + Debugf("Pushing [%s] to [%s]\n", local, remote) + + // Try to get the image + // FIXME: Handle lookup + // FIXME: Also push the tags in case of ./docker push myrepo:mytag + // img, err := srv.runtime.LookupImage(cmd.Arg(0)) + img, err := srv.runtime.graph.Get(local) + if err != nil { + Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) + + // If it fails, try to get the repository + if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { + fmt.Fprintf(stdout, "Pushing %s (%d tags) on %s...\n", local, len(localRepo), remote) + if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { + return err + } + fmt.Fprintf(stdout, "Push completed\n") + return nil + } else { + return err + } + return nil + } + fmt.Fprintf(stdout, "Pushing image %s..\n", img.Id) + err = srv.runtime.graph.PushImage(stdout, img, srv.runtime.authConfig) + if err != nil { + return err + } + fmt.Fprintf(stdout, "Push completed\n") + return nil +} + +func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "pull", "IMAGE", "Pull an image or a repository from the registry") + if err := cmd.Parse(args); err != nil { + return nil + } + remote := cmd.Arg(0) + if remote == "" { + cmd.Usage() + return nil + } + + if srv.runtime.authConfig == nil { + return fmt.Errorf("Please login prior to push. ('docker login')") + } + + if srv.runtime.graph.LookupRemoteImage(remote, srv.runtime.authConfig) { + fmt.Fprintf(stdout, "Pulling %s...\n", remote) + if err := srv.runtime.graph.PullImage(remote, srv.runtime.authConfig); err != nil { + return err + } + fmt.Fprintf(stdout, "Pulled\n") + return nil + } + // FIXME: Allow pull repo:tag + fmt.Fprintf(stdout, "Pulling %s...\n", remote) + if err := srv.runtime.graph.PullRepository(stdout, remote, "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { + return err + } + fmt.Fprintf(stdout, "Pull completed\n") + return nil +} + func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "images", "[OPTIONS] [NAME]", "List images") - limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") + //limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") quiet := cmd.Bool("q", false, "only show numeric IDs") + fl_a := cmd.Bool("a", false, "show all images") if err := cmd.Parse(args); err != nil { return nil } @@ -533,30 +507,36 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri } w := tabwriter.NewWriter(stdout, 20, 1, 3, ' ', 0) if !*quiet { - fmt.Fprintf(w, "NAME\tID\tCREATED\tPARENT\n") + fmt.Fprintf(w, "REPOSITORY\tTAG\tID\tCREATED\tPARENT\n") + } + var allImages map[string]*Image + var err error + if *fl_a { + allImages, err = srv.runtime.graph.Map() + } else { + allImages, err = srv.runtime.graph.Heads() } - paths, err := srv.images.Paths() if err != nil { return err } - for _, name := range paths { - if nameFilter != "" && nameFilter != name { + for name, repository := range srv.runtime.repositories.Repositories { + if nameFilter != "" && name != nameFilter { continue } - ids, err := srv.images.List(name) - if err != nil { - return err - } - for idx, img := range ids { - if *limit > 0 && idx >= *limit { - break + for tag, id := range repository { + image, err := srv.runtime.graph.Get(id) + if err != nil { + log.Printf("Warning: couldn't load %s from %s/%s: %s", id, name, tag, err) + continue } + delete(allImages, id) if !*quiet { for idx, field := range []string{ - /* NAME */ name, - /* ID */ img.Id, - /* CREATED */ future.HumanDuration(time.Now().Sub(time.Unix(img.Created, 0))) + " ago", - /* PARENT */ img.Parent, + /* REPOSITORY */ name, + /* TAG */ tag, + /* ID */ id, + /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", + /* PARENT */ srv.runtime.repositories.ImageName(image.Parent), } { if idx == 0 { w.Write([]byte(field)) @@ -566,7 +546,30 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri } w.Write([]byte{'\n'}) } else { - stdout.Write([]byte(img.Id + "\n")) + stdout.Write([]byte(image.Id + "\n")) + } + } + } + // Display images which aren't part of a + if nameFilter == "" { + for id, image := range allImages { + if !*quiet { + for idx, field := range []string{ + /* REPOSITORY */ "", + /* TAG */ "", + /* ID */ id, + /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", + /* PARENT */ srv.runtime.repositories.ImageName(image.Parent), + } { + if idx == 0 { + w.Write([]byte(field)) + } else { + w.Write([]byte("\t" + field)) + } + } + w.Write([]byte{'\n'}) + } else { + stdout.Write([]byte(image.Id + "\n")) } } } @@ -574,7 +577,6 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri w.Flush() } return nil - } func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { @@ -590,8 +592,7 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) if !*quiet { fmt.Fprintf(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tCOMMENT\n") } - for _, container := range srv.containers.List() { - comment := container.GetUserData("comment") + for _, container := range srv.runtime.List() { if !container.State.Running && !*fl_all { continue } @@ -602,11 +603,11 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) } for idx, field := range []string{ /* ID */ container.Id, - /* IMAGE */ container.GetUserData("image"), + /* IMAGE */ srv.runtime.repositories.ImageName(container.Image), /* COMMAND */ command, - /* CREATED */ future.HumanDuration(time.Now().Sub(container.Created)) + " ago", + /* CREATED */ HumanDuration(time.Now().Sub(container.Created)) + " ago", /* STATUS */ container.State.String(), - /* COMMENT */ comment, + /* COMMENT */ "", } { if idx == 0 { w.Write([]byte(field)) @@ -625,92 +626,36 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) return nil } -func (srv *Server) CmdLayers(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, - "layers", "[OPTIONS]", - "List filesystem layers (debug only)") - if err := cmd.Parse(args); err != nil { - return nil - } - for _, layer := range srv.images.Layers() { - fmt.Fprintln(stdout, layer) - } - return nil -} - -func (srv *Server) CmdCp(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, - "cp", "[OPTIONS] IMAGE NAME", - "Create a copy of IMAGE and call it NAME") - if err := cmd.Parse(args); err != nil { - return nil - } - if image, err := srv.images.Get(cmd.Arg(0)); err != nil { - return err - } else if image == nil { - return errors.New("Image " + cmd.Arg(0) + " does not exist") - } else { - if img, err := image.Copy(cmd.Arg(1)); err != nil { - return err - } else { - fmt.Fprintln(stdout, img.Id) - } - } - return nil -} - func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, - "commit", "[OPTIONS] CONTAINER [DEST]", + "commit", "[OPTIONS] CONTAINER [REPOSITORY [TAG]]", "Create a new image from a container's changes") if err := cmd.Parse(args); err != nil { return nil } - containerName, imgName := cmd.Arg(0), cmd.Arg(1) - if containerName == "" || imgName == "" { + containerName, repository, tag := cmd.Arg(0), cmd.Arg(1), cmd.Arg(2) + if containerName == "" { cmd.Usage() return nil } - if container := srv.containers.Get(containerName); container != nil { - // FIXME: freeze the container before copying it to avoid data corruption? - rwTar, err := fs.Tar(container.Mountpoint.Rw, fs.Uncompressed) - if err != nil { - return err - } - // Create a new image from the container's base layers + a new layer from container changes - parentImg, err := srv.images.Get(container.Image) - if err != nil { - return err - } - - img, err := srv.images.Create(rwTar, parentImg, imgName, "") - if err != nil { - return err - } - - fmt.Fprintln(stdout, img.Id) - return nil + img, err := srv.runtime.Commit(containerName, repository, tag) + if err != nil { + return err } - return errors.New("No such container: " + containerName) + fmt.Fprintln(stdout, img.Id) + return nil } -func (srv *Server) CmdTar(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +func (srv *Server) CmdExport(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, - "tar", "CONTAINER", - "Stream the contents of a container as a tar archive") - fl_sparse := cmd.Bool("s", false, "Generate a sparse tar stream (top layer + reference to bottom layers)") + "export", "CONTAINER", + "Export the contents of a filesystem as a tar archive") if err := cmd.Parse(args); err != nil { return nil } - if *fl_sparse { - return errors.New("Sparse mode not yet implemented") // FIXME - } name := cmd.Arg(0) - if container := srv.containers.Get(name); container != nil { - if err := container.Mountpoint.EnsureMounted(); err != nil { - return err - } - data, err := fs.Tar(container.Mountpoint.Root, fs.Uncompressed) + if container := srv.runtime.Get(name); container != nil { + data, err := container.Export() if err != nil { return err } @@ -733,10 +678,10 @@ func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string if cmd.NArg() < 1 { return errors.New("Not enough arguments") } - if container := srv.containers.Get(cmd.Arg(0)); container == nil { + if container := srv.runtime.Get(cmd.Arg(0)); container == nil { return errors.New("No such container") } else { - changes, err := srv.images.Changes(container.Mountpoint) + changes, err := container.Changes() if err != nil { return err } @@ -747,26 +692,6 @@ func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } -func (srv *Server) CmdReset(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, - "reset", "CONTAINER [OPTIONS]", - "Reset changes to a container's filesystem") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 1 { - return errors.New("Not enough arguments") - } - for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { - if err := container.Mountpoint.Reset(); err != nil { - return errors.New("Reset " + container.Id + ": " + err.Error()) - } - } - } - return nil -} - func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "logs", "[OPTIONS] CONTAINER", "Fetch the logs of a container") if err := cmd.Parse(args); err != nil { @@ -777,11 +702,21 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } name := cmd.Arg(0) - if container := srv.containers.Get(name); container != nil { - if _, err := io.Copy(stdout, container.StdoutLog()); err != nil { + if container := srv.runtime.Get(name); container != nil { + log_stdout, err := container.ReadLog("stdout") + if err != nil { return err } - if _, err := io.Copy(stdout, container.StderrLog()); err != nil { + log_stderr, err := container.ReadLog("stderr") + if err != nil { + return err + } + // FIXME: Interpolate stdout and stderr instead of concatenating them + // FIXME: Differentiate stdout and stderr in the remote protocol + if _, err := io.Copy(stdout, log_stdout); err != nil { + return err + } + if _, err := io.Copy(stdout, log_stderr); err != nil { return err } return nil @@ -789,31 +724,6 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return errors.New("No such container: " + cmd.Arg(0)) } -func (srv *Server) CreateContainer(img *fs.Image, ports []int, user string, tty bool, openStdin bool, memory int64, comment string, cmd string, args ...string) (*Container, error) { - id := future.RandomId()[:8] - container, err := srv.containers.Create(id, cmd, args, img, - &Config{ - Hostname: id, - Ports: ports, - User: user, - Tty: tty, - OpenStdin: openStdin, - Memory: memory, - }) - if err != nil { - return nil, err - } - if err := container.SetUserData("image", img.Id); err != nil { - srv.containers.Destroy(container) - return nil, errors.New("Error setting container userdata: " + err.Error()) - } - if err := container.SetUserData("comment", comment); err != nil { - srv.containers.Destroy(container) - return nil, errors.New("Error setting container userdata: " + err.Error()) - } - return container, nil -} - func (srv *Server) CmdAttach(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "attach", "[OPTIONS]", "Attach to a running container") fl_i := cmd.Bool("i", false, "Attach to stdin") @@ -827,7 +737,7 @@ func (srv *Server) CmdAttach(stdin io.ReadCloser, stdout io.Writer, args ...stri return nil } name := cmd.Arg(0) - container := srv.containers.Get(name) + container := srv.runtime.Get(name) if container == nil { return errors.New("No such container: " + name) } @@ -876,78 +786,54 @@ func (p *ports) Set(value string) error { return nil } -func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "run", "[OPTIONS] IMAGE COMMAND [ARG...]", "Run a command in a new container") - fl_user := cmd.String("u", "", "Username or UID") - fl_attach := cmd.Bool("a", false, "Attach stdin and stdout") - fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") - fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") - fl_comment := cmd.String("c", "", "Comment") - fl_memory := cmd.Int64("m", 0, "Memory limit (in bytes)") - var fl_ports ports +// ListOpts type +type ListOpts []string - cmd.Var(&fl_ports, "p", "Map a network port to the container") +func (opts *ListOpts) String() string { + return fmt.Sprint(*opts) +} + +func (opts *ListOpts) Set(value string) error { + *opts = append(*opts, value) + return nil +} + +func (srv *Server) CmdTag(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "tag", "[OPTIONS] IMAGE REPOSITORY [TAG]", "Tag an image into a repository") + force := cmd.Bool("f", false, "Force") if err := cmd.Parse(args); err != nil { return nil } - name := cmd.Arg(0) - var img_name string - //var img_version string // Only here for reference - var cmdline []string - - if len(cmd.Args()) >= 2 { - cmdline = cmd.Args()[1:] - } - // Choose a default image if needed - if name == "" { - name = "base" + if cmd.NArg() < 2 { + cmd.Usage() + return nil } + return srv.runtime.repositories.Set(cmd.Arg(1), cmd.Arg(2), cmd.Arg(0), *force) +} - // Choose a default command if needed - if len(cmdline) == 0 { - *fl_stdin = true - *fl_tty = true - *fl_attach = true - cmdline = []string{"/bin/bash", "-i"} - } - - // Find the image - img, err := srv.images.Find(name) +func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + config, err := ParseRun(args) if err != nil { return err - } else if img == nil { - // Separate the name:version tag - if strings.Contains(name, ":") { - parts := strings.SplitN(name, ":", 2) - img_name = parts[0] - //img_version = parts[1] // Only here for reference - } else { - img_name = name - } - - stdin_noclose := ioutil.NopCloser(stdin) - if err := srv.CmdImport(stdin_noclose, stdout, img_name); err != nil { - return err - } - img, err = srv.images.Find(name) - if err != nil || img == nil { - return errors.New("Could not find image after downloading: " + name) - } } - + if config.Image == "" { + return fmt.Errorf("Image not specified") + } + if len(config.Cmd) == 0 { + return fmt.Errorf("Command not specified") + } // Create new container - container, err := srv.CreateContainer(img, fl_ports, *fl_user, *fl_tty, - *fl_stdin, *fl_memory, *fl_comment, cmdline[0], cmdline[1:]...) + container, err := srv.runtime.Create(config) if err != nil { return errors.New("Error creating container: " + err.Error()) } - if *fl_stdin { + if config.OpenStdin { cmd_stdin, err := container.StdinPipe() if err != nil { return err } - if *fl_attach { - future.Go(func() error { + if !config.Detach { + Go(func() error { _, err := io.Copy(cmd_stdin, stdin) cmd_stdin.Close() return err @@ -955,7 +841,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) } } // Run the container - if *fl_attach { + if !config.Detach { cmd_stderr, err := container.StderrPipe() if err != nil { return err @@ -967,11 +853,11 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) if err := container.Start(); err != nil { return err } - sending_stdout := future.Go(func() error { + sending_stdout := Go(func() error { _, err := io.Copy(stdout, cmd_stdout) return err }) - sending_stderr := future.Go(func() error { + sending_stderr := Go(func() error { _, err := io.Copy(stdout, cmd_stderr) return err }) @@ -994,64 +880,20 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) } func NewServer() (*Server, error) { - future.Seed() + rand.Seed(time.Now().UTC().UnixNano()) if runtime.GOARCH != "amd64" { log.Fatalf("The docker runtime currently only supports amd64 (not %s). This will change in the future. Aborting.", runtime.GOARCH) } - // if err != nil { - // return nil, err - // } - containers, err := New() + runtime, err := NewRuntime() if err != nil { return nil, err } srv := &Server{ - images: containers.Store, - containers: containers, + runtime: runtime, } return srv, nil } -func (srv *Server) CmdMirror(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - _, err := io.Copy(stdout, stdin) - return err -} - -func (srv *Server) CmdDebug(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - for { - if line, err := bufio.NewReader(stdin).ReadString('\n'); err == nil { - fmt.Printf("--- %s", line) - } else if err == io.EOF { - if len(line) > 0 { - fmt.Printf("--- %s\n", line) - } - break - } else { - return err - } - } - return nil -} - -func (srv *Server) CmdWeb(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "web", "[OPTIONS]", "A web UI for docker") - showurl := cmd.Bool("u", false, "Return the URL of the web UI") - if err := cmd.Parse(args); err != nil { - return nil - } - if *showurl { - fmt.Fprintln(stdout, "http://localhost:4242/web") - } else { - if file, err := os.Open("dockerweb.html"); err != nil { - return err - } else if _, err := io.Copy(stdout, file); err != nil { - return err - } - } - return nil -} - type Server struct { - containers *Docker - images *fs.Store + runtime *Runtime } diff --git a/container.go b/container.go index c80129e2a8..f900599d00 100644 --- a/container.go +++ b/container.go @@ -3,7 +3,8 @@ package docker import ( "encoding/json" "errors" - "github.com/dotcloud/docker/fs" + "flag" + "fmt" "github.com/kr/pty" "io" "io/ioutil" @@ -16,40 +17,33 @@ import ( "time" ) -var sysInitPath string - -func init() { - sysInitPath = SelfPath() -} - type Container struct { - Id string - Root string + root string + + Id string Created time.Time Path string Args []string - Config *Config - Mountpoint *fs.Mountpoint - State *State - Image string + Config *Config + State State + Image string network *NetworkInterface - networkManager *NetworkManager NetworkSettings *NetworkSettings - SysInitPath string - lxcConfigPath string - cmd *exec.Cmd - stdout *writeBroadcaster - stderr *writeBroadcaster - stdin io.ReadCloser - stdinPipe io.WriteCloser + SysInitPath string + cmd *exec.Cmd + stdout *writeBroadcaster + stderr *writeBroadcaster + stdin io.ReadCloser + stdinPipe io.WriteCloser stdoutLog *os.File stderrLog *os.File + runtime *Runtime } type Config struct { @@ -57,9 +51,43 @@ type Config struct { User string Memory int64 // Memory limit (in bytes) MemorySwap int64 // Total memory usage (memory + swap); set `-1' to disable swap + Detach bool Ports []int Tty bool // Attach standard streams to a tty, including stdin if it is not closed. OpenStdin bool // Open stdin + Env []string + Cmd []string + Image string // Name of the image as it was passed by the operator (eg. could be symbolic) +} + +func ParseRun(args []string) (*Config, error) { + cmd := flag.NewFlagSet("", flag.ContinueOnError) + cmd.SetOutput(ioutil.Discard) + fl_user := cmd.String("u", "", "Username or UID") + fl_detach := cmd.Bool("d", false, "Detached mode: leave the container running in the background") + fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") + fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") + fl_memory := cmd.Int64("m", 0, "Memory limit (in bytes)") + var fl_ports ports + + cmd.Var(&fl_ports, "p", "Map a network port to the container") + var fl_env ListOpts + cmd.Var(&fl_env, "e", "Set environment variables") + if err := cmd.Parse(args); err != nil { + return nil, err + } + config := &Config{ + Ports: fl_ports, + User: *fl_user, + Tty: *fl_tty, + OpenStdin: *fl_stdin, + Memory: *fl_memory, + Detach: *fl_detach, + Env: fl_env, + Cmd: cmd.Args()[1:], + Image: cmd.Arg(0), + } + return config, nil } type NetworkSettings struct { @@ -69,106 +97,6 @@ type NetworkSettings struct { PortMapping map[string]string } -func createContainer(id string, root string, command string, args []string, image *fs.Image, config *Config, netManager *NetworkManager) (*Container, error) { - mountpoint, err := image.Mountpoint(path.Join(root, "rootfs"), path.Join(root, "rw")) - if err != nil { - return nil, err - } - container := &Container{ - Id: id, - Root: root, - Created: time.Now(), - Path: command, - Args: args, - Config: config, - Image: image.Id, - Mountpoint: mountpoint, - State: newState(), - networkManager: netManager, - NetworkSettings: &NetworkSettings{}, - SysInitPath: sysInitPath, - lxcConfigPath: path.Join(root, "config.lxc"), - stdout: newWriteBroadcaster(), - stderr: newWriteBroadcaster(), - } - if err := os.Mkdir(root, 0700); err != nil { - return nil, err - } - // Setup logging of stdout and stderr to disk - if stdoutLog, err := os.OpenFile(path.Join(container.Root, id+"-stdout.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { - return nil, err - } else { - container.stdoutLog = stdoutLog - } - if stderrLog, err := os.OpenFile(path.Join(container.Root, id+"-stderr.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { - return nil, err - } else { - container.stderrLog = stderrLog - } - if container.Config.OpenStdin { - container.stdin, container.stdinPipe = io.Pipe() - } else { - container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin - } - container.stdout.AddWriter(NopWriteCloser(container.stdoutLog)) - container.stderr.AddWriter(NopWriteCloser(container.stderrLog)) - - if err := container.save(); err != nil { - return nil, err - } - return container, nil -} - -func loadContainer(store *fs.Store, containerPath string, netManager *NetworkManager) (*Container, error) { - data, err := ioutil.ReadFile(path.Join(containerPath, "config.json")) - if err != nil { - return nil, err - } - mountpoint, err := store.FetchMountpoint( - path.Join(containerPath, "rootfs"), - path.Join(containerPath, "rw"), - ) - if err != nil { - return nil, err - } else if mountpoint == nil { - return nil, errors.New("Couldn't load container: unregistered mountpoint.") - } - container := &Container{ - stdout: newWriteBroadcaster(), - stderr: newWriteBroadcaster(), - lxcConfigPath: path.Join(containerPath, "config.lxc"), - networkManager: netManager, - NetworkSettings: &NetworkSettings{}, - Mountpoint: mountpoint, - } - // Load container settings - if err := json.Unmarshal(data, container); err != nil { - return nil, err - } - - // Setup logging of stdout and stderr to disk - if stdoutLog, err := os.OpenFile(path.Join(container.Root, container.Id+"-stdout.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { - return nil, err - } else { - container.stdoutLog = stdoutLog - } - if stderrLog, err := os.OpenFile(path.Join(container.Root, container.Id+"-stderr.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { - return nil, err - } else { - container.stderrLog = stderrLog - } - container.stdout.AddWriter(NopWriteCloser(container.stdoutLog)) - container.stderr.AddWriter(NopWriteCloser(container.stderrLog)) - - if container.Config.OpenStdin { - container.stdin, container.stdinPipe = io.Pipe() - } else { - container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin - } - container.State = newState() - return container, nil -} - func (container *Container) Cmd() *exec.Cmd { return container.cmd } @@ -177,64 +105,32 @@ func (container *Container) When() time.Time { return container.Created } -func (container *Container) loadUserData() (map[string]string, error) { - jsonData, err := ioutil.ReadFile(path.Join(container.Root, "userdata.json")) - if err != nil { - if os.IsNotExist(err) { - return make(map[string]string), nil - } - return nil, err - } - data := make(map[string]string) - if err := json.Unmarshal(jsonData, &data); err != nil { - return nil, err - } - return data, nil -} - -func (container *Container) saveUserData(data map[string]string) error { - jsonData, err := json.Marshal(data) +func (container *Container) FromDisk() error { + data, err := ioutil.ReadFile(container.jsonPath()) if err != nil { return err } - return ioutil.WriteFile(path.Join(container.Root, "userdata.json"), jsonData, 0700) -} - -func (container *Container) SetUserData(key, value string) error { - data, err := container.loadUserData() - if err != nil { + // Load container settings + if err := json.Unmarshal(data, container); err != nil { return err } - data[key] = value - return container.saveUserData(data) + return nil } -func (container *Container) GetUserData(key string) string { - data, err := container.loadUserData() - if err != nil { - return "" - } - if value, exists := data[key]; exists { - return value - } - return "" -} - -func (container *Container) save() (err error) { +func (container *Container) ToDisk() (err error) { data, err := json.Marshal(container) if err != nil { return } - return ioutil.WriteFile(path.Join(container.Root, "config.json"), data, 0666) + return ioutil.WriteFile(container.jsonPath(), data, 0666) } func (container *Container) generateLXCConfig() error { - fo, err := os.Create(container.lxcConfigPath) + fo, err := os.Create(container.lxcConfigPath()) if err != nil { return err } defer fo.Close() - if err := LxcTemplateCompiled.Execute(fo, container); err != nil { return err } @@ -309,7 +205,7 @@ func (container *Container) start() error { } func (container *Container) Start() error { - if err := container.Mountpoint.EnsureMounted(); err != nil { + if err := container.EnsureMounted(); err != nil { return err } if err := container.allocateNetwork(); err != nil { @@ -320,7 +216,7 @@ func (container *Container) Start() error { } params := []string{ "-n", container.Id, - "-f", container.lxcConfigPath, + "-f", container.lxcConfigPath(), "--", "/sbin/init", } @@ -339,6 +235,15 @@ func (container *Container) Start() error { container.cmd = exec.Command("/usr/bin/lxc-start", params...) + // Setup environment + container.cmd.Env = append( + []string{ + "HOME=/", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + container.Config.Env..., + ) + var err error if container.Config.Tty { err = container.startPty() @@ -348,8 +253,10 @@ func (container *Container) Start() error { if err != nil { return err } + // FIXME: save state on disk *first*, then converge + // this way disk state is used as a journal, eg. we can restore after crash etc. container.State.setRunning(container.cmd.Process.Pid) - container.save() + container.ToDisk() go container.monitor() return nil } @@ -389,30 +296,14 @@ func (container *Container) StdoutPipe() (io.ReadCloser, error) { return newBufReader(reader), nil } -func (container *Container) StdoutLog() io.Reader { - r, err := os.Open(container.stdoutLog.Name()) - if err != nil { - return nil - } - return r -} - func (container *Container) StderrPipe() (io.ReadCloser, error) { reader, writer := io.Pipe() container.stderr.AddWriter(writer) return newBufReader(reader), nil } -func (container *Container) StderrLog() io.Reader { - r, err := os.Open(container.stderrLog.Name()) - if err != nil { - return nil - } - return r -} - func (container *Container) allocateNetwork() error { - iface, err := container.networkManager.Allocate() + iface, err := container.runtime.networkManager.Allocate() if err != nil { return err } @@ -450,7 +341,7 @@ func (container *Container) monitor() { } container.stdout.Close() container.stderr.Close() - if err := container.Mountpoint.Umount(); err != nil { + if err := container.Unmount(); err != nil { log.Printf("%v: Failed to umount filesystem: %v", container.Id, err) } @@ -461,7 +352,7 @@ func (container *Container) monitor() { // Report status back container.State.setStopped(exitCode) - container.save() + container.ToDisk() } func (container *Container) kill() error { @@ -523,6 +414,17 @@ func (container *Container) Wait() int { return container.State.ExitCode } +func (container *Container) ExportRw() (Archive, error) { + return Tar(container.rwPath(), Uncompressed) +} + +func (container *Container) Export() (Archive, error) { + if err := container.EnsureMounted(); err != nil { + return nil, err + } + return Tar(container.RootfsPath(), Uncompressed) +} + func (container *Container) WaitTimeout(timeout time.Duration) error { done := make(chan bool) go func() { @@ -538,3 +440,75 @@ func (container *Container) WaitTimeout(timeout time.Duration) error { } return nil } + +func (container *Container) EnsureMounted() error { + if mounted, err := container.Mounted(); err != nil { + return err + } else if mounted { + return nil + } + return container.Mount() +} + +func (container *Container) Mount() error { + image, err := container.GetImage() + if err != nil { + return err + } + return image.Mount(container.RootfsPath(), container.rwPath()) +} + +func (container *Container) Changes() ([]Change, error) { + image, err := container.GetImage() + if err != nil { + return nil, err + } + return image.Changes(container.rwPath()) +} + +func (container *Container) GetImage() (*Image, error) { + if container.runtime == nil { + return nil, fmt.Errorf("Can't get image of unregistered container") + } + return container.runtime.graph.Get(container.Image) +} + +func (container *Container) Mounted() (bool, error) { + return Mounted(container.RootfsPath()) +} + +func (container *Container) Unmount() error { + return Unmount(container.RootfsPath()) +} + +func (container *Container) logPath(name string) string { + return path.Join(container.root, fmt.Sprintf("%s-%s.log", container.Id, name)) +} + +func (container *Container) ReadLog(name string) (io.Reader, error) { + return os.Open(container.logPath(name)) +} + +func (container *Container) jsonPath() string { + return path.Join(container.root, "config.json") +} + +func (container *Container) lxcConfigPath() string { + return path.Join(container.root, "config.lxc") +} + +// This method must be exported to be used from the lxc template +func (container *Container) RootfsPath() string { + return path.Join(container.root, "rootfs") +} + +func (container *Container) rwPath() string { + return path.Join(container.root, "rw") +} + +func validateId(id string) error { + if id == "" { + return fmt.Errorf("Invalid empty id") + } + return nil +} diff --git a/container_test.go b/container_test.go index 18beeba253..3f9b02ab58 100644 --- a/container_test.go +++ b/container_test.go @@ -3,7 +3,6 @@ package docker import ( "bufio" "fmt" - "github.com/dotcloud/docker/fs" "io" "io/ioutil" "math/rand" @@ -15,24 +14,22 @@ import ( ) func TestCommitRun(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container1, err := docker.Create( - "precommit_test", - "/bin/sh", - []string{"-c", "echo hello > /world"}, - GetTestImage(docker), + defer nuke(runtime) + container1, err := runtime.Create( &Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"/bin/sh", "-c", "echo hello > /world"}, Memory: 33554432, }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container1) + defer runtime.Destroy(container1) if container1.State.Running { t.Errorf("Container shouldn't be running") @@ -44,37 +41,28 @@ func TestCommitRun(t *testing.T) { t.Errorf("Container shouldn't be running") } - // FIXME: freeze the container before copying it to avoid data corruption? - rwTar, err := fs.Tar(container1.Mountpoint.Rw, fs.Uncompressed) + rwTar, err := container1.ExportRw() if err != nil { t.Error(err) } - // Create a new image from the container's base layers + a new layer from container changes - parentImg, err := docker.Store.Get(container1.Image) - if err != nil { - t.Error(err) - } - - img, err := docker.Store.Create(rwTar, parentImg, "test_commitrun", "unit test commited image") + img, err := runtime.graph.Create(rwTar, container1, "unit test commited image") if err != nil { t.Error(err) } // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world - container2, err := docker.Create( - "postcommit_test", - "cat", - []string{"/world"}, - img, + container2, err := runtime.Create( &Config{ + Image: img.Id, Memory: 33554432, + Cmd: []string{"cat", "/world"}, }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container2) + defer runtime.Destroy(container2) stdout, err := container2.StdoutPipe() stderr, err := container2.StderrPipe() @@ -92,24 +80,22 @@ func TestCommitRun(t *testing.T) { } func TestRun(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( - "run_test", - "ls", - []string{"-al"}, - GetTestImage(docker), + defer nuke(runtime) + container, err := runtime.Create( &Config{ + Image: GetTestImage(runtime).Id, Memory: 33554432, + Cmd: []string{"ls", "-al"}, }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) if container.State.Running { t.Errorf("Container shouldn't be running") @@ -123,22 +109,21 @@ func TestRun(t *testing.T) { } func TestOutput(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( - "output_test", - "echo", - []string{"-n", "foobar"}, - GetTestImage(docker), - &Config{}, + defer nuke(runtime) + container, err := runtime.Create( + &Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"echo", "-n", "foobar"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err := container.Output() if err != nil { t.Fatal(err) @@ -149,22 +134,20 @@ func TestOutput(t *testing.T) { } func TestKill(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( - "stop_test", - "cat", - []string{"/dev/zero"}, - GetTestImage(docker), - &Config{}, + defer nuke(runtime) + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat", "/dev/zero"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) if container.State.Running { t.Errorf("Container shouldn't be running") @@ -192,38 +175,35 @@ func TestKill(t *testing.T) { } func TestExitCode(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) - trueContainer, err := docker.Create( - "exit_test_1", - "/bin/true", - []string{""}, - GetTestImage(docker), - &Config{}, + trueContainer, err := runtime.Create(&Config{ + + Image: GetTestImage(runtime).Id, + Cmd: []string{"/bin/true", ""}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(trueContainer) + defer runtime.Destroy(trueContainer) if err := trueContainer.Run(); err != nil { t.Fatal(err) } - falseContainer, err := docker.Create( - "exit_test_2", - "/bin/false", - []string{""}, - GetTestImage(docker), - &Config{}, + falseContainer, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"/bin/false", ""}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(falseContainer) + defer runtime.Destroy(falseContainer) if err := falseContainer.Run(); err != nil { t.Fatal(err) } @@ -238,22 +218,20 @@ func TestExitCode(t *testing.T) { } func TestRestart(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( - "restart_test", - "echo", - []string{"-n", "foobar"}, - GetTestImage(docker), - &Config{}, + defer nuke(runtime) + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"echo", "-n", "foobar"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err := container.Output() if err != nil { t.Fatal(err) @@ -273,24 +251,22 @@ func TestRestart(t *testing.T) { } func TestRestartStdin(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( - "restart_stdin_test", - "cat", - []string{}, - GetTestImage(docker), - &Config{ - OpenStdin: true, - }, + defer nuke(runtime) + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat"}, + + OpenStdin: true, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) stdin, err := container.StdinPipe() stdout, err := container.StdoutPipe() @@ -323,24 +299,22 @@ func TestRestartStdin(t *testing.T) { } func TestUser(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) // Default user must be root - container, err := docker.Create( - "user_default", - "id", - []string{}, - GetTestImage(docker), - &Config{}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err := container.Output() if err != nil { t.Fatal(err) @@ -350,19 +324,17 @@ func TestUser(t *testing.T) { } // Set a username - container, err = docker.Create( - "user_root", - "id", - []string{}, - GetTestImage(docker), - &Config{ - User: "root", - }, + container, err = runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, + + User: "root", + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err = container.Output() if err != nil || container.State.ExitCode != 0 { t.Fatal(err) @@ -372,19 +344,17 @@ func TestUser(t *testing.T) { } // Set a UID - container, err = docker.Create( - "user_uid0", - "id", - []string{}, - GetTestImage(docker), - &Config{ - User: "0", - }, + container, err = runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, + + User: "0", + }, ) if err != nil || container.State.ExitCode != 0 { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err = container.Output() if err != nil || container.State.ExitCode != 0 { t.Fatal(err) @@ -394,19 +364,17 @@ func TestUser(t *testing.T) { } // Set a different user by uid - container, err = docker.Create( - "user_uid1", - "id", - []string{}, - GetTestImage(docker), - &Config{ - User: "1", - }, + container, err = runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, + + User: "1", + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err = container.Output() if err != nil { t.Fatal(err) @@ -418,19 +386,17 @@ func TestUser(t *testing.T) { } // Set a different user by username - container, err = docker.Create( - "user_daemon", - "id", - []string{}, - GetTestImage(docker), - &Config{ - User: "daemon", - }, + container, err = runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, + + User: "daemon", + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err = container.Output() if err != nil || container.State.ExitCode != 0 { t.Fatal(err) @@ -441,35 +407,31 @@ func TestUser(t *testing.T) { } func TestMultipleContainers(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) - container1, err := docker.Create( - "container1", - "cat", - []string{"/dev/zero"}, - GetTestImage(docker), - &Config{}, + container1, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat", "/dev/zero"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container1) + defer runtime.Destroy(container1) - container2, err := docker.Create( - "container2", - "cat", - []string{"/dev/zero"}, - GetTestImage(docker), - &Config{}, + container2, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat", "/dev/zero"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container2) + defer runtime.Destroy(container2) // Start both containers if err := container1.Start(); err != nil { @@ -498,24 +460,22 @@ func TestMultipleContainers(t *testing.T) { } func TestStdin(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( - "stdin_test", - "cat", - []string{}, - GetTestImage(docker), - &Config{ - OpenStdin: true, - }, + defer nuke(runtime) + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat"}, + + OpenStdin: true, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) stdin, err := container.StdinPipe() stdout, err := container.StdoutPipe() @@ -534,24 +494,22 @@ func TestStdin(t *testing.T) { } func TestTty(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( - "tty_test", - "cat", - []string{}, - GetTestImage(docker), - &Config{ - OpenStdin: true, - }, + defer nuke(runtime) + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat"}, + + OpenStdin: true, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) stdin, err := container.StdinPipe() stdout, err := container.StdoutPipe() @@ -570,22 +528,20 @@ func TestTty(t *testing.T) { } func TestEnv(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( - "env_test", - "/usr/bin/env", - []string{}, - GetTestImage(docker), - &Config{}, + defer nuke(runtime) + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"/usr/bin/env"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) stdout, err := container.StdoutPipe() if err != nil { t.Fatal(err) @@ -640,56 +596,52 @@ func grepFile(t *testing.T, path string, pattern string) { } func TestLXCConfig(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) // Memory is allocated randomly for testing rand.Seed(time.Now().UTC().UnixNano()) memMin := 33554432 memMax := 536870912 mem := memMin + rand.Intn(memMax-memMin) - container, err := docker.Create( - "config_test", - "/bin/true", - []string{}, - GetTestImage(docker), - &Config{ - Hostname: "foobar", - Memory: int64(mem), - }, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"/bin/true"}, + + Hostname: "foobar", + Memory: int64(mem), + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) container.generateLXCConfig() - grepFile(t, container.lxcConfigPath, "lxc.utsname = foobar") - grepFile(t, container.lxcConfigPath, + grepFile(t, container.lxcConfigPath(), "lxc.utsname = foobar") + grepFile(t, container.lxcConfigPath(), fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", mem)) - grepFile(t, container.lxcConfigPath, + grepFile(t, container.lxcConfigPath(), fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", mem*2)) } func BenchmarkRunSequencial(b *testing.B) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { b.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) for i := 0; i < b.N; i++ { - container, err := docker.Create( - fmt.Sprintf("bench_%v", i), - "echo", - []string{"-n", "foo"}, - GetTestImage(docker), - &Config{}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"echo", "-n", "foo"}, + }, ) if err != nil { b.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err := container.Output() if err != nil { b.Fatal(err) @@ -697,18 +649,18 @@ func BenchmarkRunSequencial(b *testing.B) { if string(output) != "foo" { b.Fatalf("Unexecpted output: %v", string(output)) } - if err := docker.Destroy(container); err != nil { + if err := runtime.Destroy(container); err != nil { b.Fatal(err) } } } func BenchmarkRunParallel(b *testing.B) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { b.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) var tasks []chan error @@ -716,18 +668,16 @@ func BenchmarkRunParallel(b *testing.B) { complete := make(chan error) tasks = append(tasks, complete) go func(i int, complete chan error) { - container, err := docker.Create( - fmt.Sprintf("bench_%v", i), - "echo", - []string{"-n", "foo"}, - GetTestImage(docker), - &Config{}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"echo", "-n", "foo"}, + }, ) if err != nil { complete <- err return } - defer docker.Destroy(container) + defer runtime.Destroy(container) if err := container.Start(); err != nil { complete <- err return @@ -739,7 +689,7 @@ func BenchmarkRunParallel(b *testing.B) { // if string(output) != "foo" { // complete <- fmt.Errorf("Unexecpted output: %v", string(output)) // } - if err := docker.Destroy(container); err != nil { + if err := runtime.Destroy(container); err != nil { complete <- err return } diff --git a/Makefile b/deb/Makefile.deb similarity index 83% rename from Makefile rename to deb/Makefile.deb index 2178ebcb93..c954b0f5b5 100644 --- a/Makefile +++ b/deb/Makefile.deb @@ -8,7 +8,7 @@ GITHUB_PATH=src/github.com/dotcloud/docker INSDIR=usr/bin SOURCE_PACKAGE=$(PKG_NAME)_$(PKG_VERSION).orig.tar.gz DEB_PACKAGE=$(PKG_NAME)_$(PKG_VERSION)_$(PKG_ARCH).deb -EXTRA_GO_PKG=fs auth +EXTRA_GO_PKG=./auth TMPDIR=$(shell mktemp -d -t XXXXXX) @@ -63,20 +63,10 @@ build_local: gotest: @echo "\033[36m[Testing]\033[00m docker..." - @sudo -E GOPATH=$(ROOT_PATH)/$(BUILD_SRC) go test -v && \ + @sudo -E GOPATH=$(ROOT_PATH)/$(BUILD_SRC) go test -v . $(EXTRA_GO_PKG) && \ echo -n "\033[32m[OK]\033[00m" || \ echo -n "\033[31m[FAIL]\033[00m"; \ echo " docker" - @echo "Testing extra repos {$(EXTRA_GO_PKG)}" - @for package in $(EXTRA_GO_PKG); do \ - echo "\033[36m[Testing]\033[00m docker/$$package..." && \ - cd $$package ; \ - sudo -E GOPATH=$(ROOT_PATH)/$(BUILD_SRC) go test -v && \ - echo -n "\033[32m[OK]\033[00m" || \ - echo -n "\033[31m[FAIL]\033[00m" ; \ - echo " docker/$$package" ; \ - cd .. ;\ - done @sudo rm -rf /tmp/docker-* clean: diff --git a/docker.go b/docker.go deleted file mode 100644 index 626c7452ea..0000000000 --- a/docker.go +++ /dev/null @@ -1,161 +0,0 @@ -package docker - -import ( - "container/list" - "fmt" - "github.com/dotcloud/docker/fs" - "io/ioutil" - "log" - "os" - "path" - "sort" -) - -type Docker struct { - root string - repository string - containers *list.List - networkManager *NetworkManager - Store *fs.Store -} - -func (docker *Docker) List() []*Container { - containers := new(History) - for e := docker.containers.Front(); e != nil; e = e.Next() { - containers.Add(e.Value.(*Container)) - } - return *containers -} - -func (docker *Docker) getContainerElement(id string) *list.Element { - for e := docker.containers.Front(); e != nil; e = e.Next() { - container := e.Value.(*Container) - if container.Id == id { - return e - } - } - return nil -} - -func (docker *Docker) Get(id string) *Container { - e := docker.getContainerElement(id) - if e == nil { - return nil - } - return e.Value.(*Container) -} - -func (docker *Docker) Exists(id string) bool { - return docker.Get(id) != nil -} - -func (docker *Docker) Create(id string, command string, args []string, image *fs.Image, config *Config) (*Container, error) { - if docker.Exists(id) { - return nil, fmt.Errorf("Container %v already exists", id) - } - root := path.Join(docker.repository, id) - - container, err := createContainer(id, root, command, args, image, config, docker.networkManager) - if err != nil { - return nil, err - } - docker.containers.PushBack(container) - return container, nil -} - -func (docker *Docker) Destroy(container *Container) error { - element := docker.getContainerElement(container.Id) - if element == nil { - return fmt.Errorf("Container %v not found - maybe it was already destroyed?", container.Id) - } - - if err := container.Stop(); err != nil { - return err - } - if container.Mountpoint.Mounted() { - if err := container.Mountpoint.Umount(); err != nil { - return fmt.Errorf("Unable to umount container %v: %v", container.Id, err) - } - } - if err := container.Mountpoint.Deregister(); err != nil { - return fmt.Errorf("Unable to deregiser -- ? mountpoint %v: %v", container.Mountpoint.Root, err) - } - if err := os.RemoveAll(container.Root); err != nil { - return fmt.Errorf("Unable to remove filesystem for %v: %v", container.Id, err) - } - docker.containers.Remove(element) - return nil -} - -func (docker *Docker) restore() error { - dir, err := ioutil.ReadDir(docker.repository) - if err != nil { - return err - } - for _, v := range dir { - container, err := loadContainer(docker.Store, path.Join(docker.repository, v.Name()), docker.networkManager) - if err != nil { - log.Printf("Failed to load container %v: %v", v.Name(), err) - continue - } - docker.containers.PushBack(container) - } - return nil -} - -func New() (*Docker, error) { - return NewFromDirectory("/var/lib/docker") -} - -func NewFromDirectory(root string) (*Docker, error) { - docker_repo := path.Join(root, "containers") - - if err := os.MkdirAll(docker_repo, 0700); err != nil && !os.IsExist(err) { - return nil, err - } - - store, err := fs.New(path.Join(root, "images")) - if err != nil { - return nil, err - } - netManager, err := newNetworkManager(networkBridgeIface) - if err != nil { - return nil, err - } - - docker := &Docker{ - root: root, - repository: docker_repo, - containers: list.New(), - Store: store, - networkManager: netManager, - } - - if err := docker.restore(); err != nil { - return nil, err - } - return docker, nil -} - -type History []*Container - -func (history *History) Len() int { - return len(*history) -} - -func (history *History) Less(i, j int) bool { - containers := *history - return containers[j].When().Before(containers[i].When()) -} - -func (history *History) Swap(i, j int) { - containers := *history - tmp := containers[i] - containers[i] = containers[j] - containers[j] = tmp -} - -func (history *History) Add(container *Container) { - *history = append(*history, container) - sort.Sort(history) -} diff --git a/docker/docker.go b/docker/docker.go index 79fe582e97..686cd2181a 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -3,7 +3,6 @@ package main import ( "flag" "github.com/dotcloud/docker" - "github.com/dotcloud/docker/future" "github.com/dotcloud/docker/rcli" "github.com/dotcloud/docker/term" "io" @@ -17,8 +16,11 @@ func main() { docker.SysInit() return } + // FIXME: Switch d and D ? (to be more sshd like) fl_daemon := flag.Bool("d", false, "Daemon mode") + fl_debug := flag.Bool("D", false, "Debug mode") flag.Parse() + rcli.DEBUG_FLAG = *fl_debug if *fl_daemon { if flag.NArg() != 0 { flag.Usage() @@ -57,11 +59,11 @@ func runCommand(args []string) error { // closing the connection. // See http://code.google.com/p/go/issues/detail?id=3345 if conn, err := rcli.Call("tcp", "127.0.0.1:4242", args...); err == nil { - receive_stdout := future.Go(func() error { + receive_stdout := docker.Go(func() error { _, err := io.Copy(os.Stdout, conn) return err }) - send_stdin := future.Go(func() error { + send_stdin := docker.Go(func() error { _, err := io.Copy(conn, os.Stdin) if err := conn.CloseWrite(); err != nil { log.Printf("Couldn't send EOF: " + err.Error()) diff --git a/fs/layers.go b/fs/layers.go deleted file mode 100644 index dc7e621f85..0000000000 --- a/fs/layers.go +++ /dev/null @@ -1,113 +0,0 @@ -package fs - -import ( - "errors" - "fmt" - "github.com/dotcloud/docker/future" - "io/ioutil" - "os" - "path" - "path/filepath" -) - -type LayerStore struct { - Root string -} - -func NewLayerStore(root string) (*LayerStore, error) { - abspath, err := filepath.Abs(root) - if err != nil { - return nil, err - } - // Create the root directory if it doesn't exists - if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { - return nil, err - } - return &LayerStore{ - Root: abspath, - }, nil -} - -func (store *LayerStore) List() []string { - files, err := ioutil.ReadDir(store.Root) - if err != nil { - return []string{} - } - var layers []string - for _, st := range files { - if st.IsDir() { - layers = append(layers, path.Join(store.Root, st.Name())) - } - } - return layers -} - -func (store *LayerStore) Get(id string) string { - if !store.Exists(id) { - return "" - } - return store.layerPath(id) -} - -func (store *LayerStore) rootExists() (bool, error) { - if stat, err := os.Stat(store.Root); err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } else if !stat.IsDir() { - return false, errors.New("Not a directory: " + store.Root) - } - return true, nil -} - -func (store *LayerStore) Init() error { - if exists, err := store.rootExists(); err != nil { - return err - } else if exists { - return nil - } - return os.Mkdir(store.Root, 0700) -} - -func (store *LayerStore) Mktemp() (string, error) { - tmpName := future.RandomId() - tmpPath := path.Join(store.Root, "tmp-"+tmpName) - if err := os.Mkdir(tmpPath, 0700); err != nil { - return "", err - } - return tmpPath, nil -} - -func (store *LayerStore) layerPath(id string) string { - return path.Join(store.Root, id) -} - -func (store *LayerStore) AddLayer(id string, archive Archive) (string, error) { - if _, err := os.Stat(store.layerPath(id)); err == nil { - return "", fmt.Errorf("Layer already exists: %v", id) - } - tmp, err := store.Mktemp() - defer os.RemoveAll(tmp) - if err != nil { - return "", fmt.Errorf("Mktemp failed: %s", err) - } - if err := Untar(archive, tmp); err != nil { - return "", err - } - layer := store.layerPath(id) - if !store.Exists(id) { - if err := os.Rename(tmp, layer); err != nil { - return "", fmt.Errorf("Could not rename temp dir to layer %s: %s", layer, err) - } - } - return layer, nil -} - -func (store *LayerStore) Exists(id string) bool { - st, err := os.Stat(store.layerPath(id)) - if err != nil { - return false - } - return st.IsDir() -} diff --git a/fs/layers_test.go b/fs/layers_test.go deleted file mode 100644 index ee7e59fd23..0000000000 --- a/fs/layers_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package fs - -import ( - "archive/tar" - "bytes" - "io" - "io/ioutil" - "os" - "testing" -) - -func fakeTar() (io.Reader, error) { - content := []byte("Hello world!\n") - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { - hdr := new(tar.Header) - hdr.Size = int64(len(content)) - hdr.Name = name - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - tw.Write([]byte(content)) - } - tw.Close() - return buf, nil -} - -func TestLayersInit(t *testing.T) { - store := tempStore(t) - defer os.RemoveAll(store.Root) - // Root should exist - if _, err := os.Stat(store.Root); err != nil { - t.Fatal(err) - } - // List() should be empty - if l := store.List(); len(l) != 0 { - t.Fatalf("List() should return %d, not %d", 0, len(l)) - } -} - -func TestAddLayer(t *testing.T) { - store := tempStore(t) - defer os.RemoveAll(store.Root) - layer, err := store.AddLayer("foo", testArchive(t)) - if err != nil { - t.Fatal(err) - } - // Layer path should exist - if _, err := os.Stat(layer); err != nil { - t.Fatal(err) - } - // List() should return 1 layer - if l := store.List(); len(l) != 1 { - t.Fatalf("List() should return %d elements, not %d", 1, len(l)) - } - // Get("foo") should return the correct layer - if foo := store.Get("foo"); foo != layer { - t.Fatalf("get(\"foo\") should return '%d', not '%d'", layer, foo) - } -} - -func TestAddLayerDuplicate(t *testing.T) { - store := tempStore(t) - defer os.RemoveAll(store.Root) - if _, err := store.AddLayer("foobar123", testArchive(t)); err != nil { - t.Fatal(err) - } - if _, err := store.AddLayer("foobar123", testArchive(t)); err == nil { - t.Fatalf("Creating duplicate layer should fail") - } -} - -/* - * HELPER FUNCTIONS - */ - -func tempStore(t *testing.T) *LayerStore { - tmp, err := ioutil.TempDir("", "docker-fs-layerstore-") - if err != nil { - t.Fatal(err) - } - store, err := NewLayerStore(tmp) - if err != nil { - t.Fatal(err) - } - return store -} - -func testArchive(t *testing.T) Archive { - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - return archive -} diff --git a/fs/mount_darwin.go b/fs/mount_darwin.go deleted file mode 100644 index 540d6f7691..0000000000 --- a/fs/mount_darwin.go +++ /dev/null @@ -1,7 +0,0 @@ -package fs - -import "errors" - -func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { - return errors.New("mount is not implemented on darwin") -} diff --git a/fs/mount_linux.go b/fs/mount_linux.go deleted file mode 100644 index b36888f75c..0000000000 --- a/fs/mount_linux.go +++ /dev/null @@ -1,7 +0,0 @@ -package fs - -import "syscall" - -func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { - return syscall.Mount(source, target, fstype, flags, data) -} diff --git a/fs/remove_test.go b/fs/remove_test.go deleted file mode 100644 index 81a234decf..0000000000 --- a/fs/remove_test.go +++ /dev/null @@ -1,222 +0,0 @@ -package fs - -import ( - "fmt" - "testing" -) - -func countImages(store *Store) int { - paths, err := store.Images() - if err != nil { - panic(err) - } - return len(paths) -} - -func TestRemoveInPath(t *testing.T) { - store, err := TempStore("test-remove-in-path") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create / Delete all - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, "foo", "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveInPath("foo"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create / Delete 1 - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveInPath("foo-0"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 9 { - t.Fatalf("Expected 9 images, %d found", c) - } - - // Delete failure - if err := store.RemoveInPath("Not_Foo"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 9 { - t.Fatalf("Expected 9 images, %d found", c) - } -} - -func TestRemove(t *testing.T) { - store, err := TempStore("test-remove") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 1 create / 1 delete - img, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 1 { - t.Fatalf("Expected 1 images, %d found", c) - } - if err := store.Remove(img); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 2 create (same name) / 1 delete - img1, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - img2, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 2 { - t.Fatalf("Expected 2 images, %d found", c) - } - if err := store.Remove(img1); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 1 { - t.Fatalf("Expected 1 images, %d found", c) - } - - // Test delete wrong name - // Note: If we change orm and Delete of non existing return error, we will need to change this test - if err := store.Remove(&Image{Id: "Not_foo", store: img2.store}); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 1 { - t.Fatalf("Expected 1 images, %d found", c) - } - - // Test delete last one - if err := store.Remove(img2); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } -} - -func TestRemoveRegexp(t *testing.T) { - store, err := TempStore("test-remove-regexp") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create with different names / Delete all good regexp - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveRegexp("foo"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create with different names / Delete all good regexp globing - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveRegexp("foo-*"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create with different names / Delete all bad regexp - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveRegexp("oo-*"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create with different names / Delete none strict regexp - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveRegexp("^oo-"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - - // Test delete 2 - if err := store.RemoveRegexp("^foo-[1,2]$"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 8 { - t.Fatalf("Expected 8 images, %d found", c) - } -} diff --git a/fs/store.go b/fs/store.go deleted file mode 100644 index d7fcb35421..0000000000 --- a/fs/store.go +++ /dev/null @@ -1,521 +0,0 @@ -package fs - -import ( - "database/sql" - "fmt" - "github.com/dotcloud/docker/future" - _ "github.com/mattn/go-sqlite3" - "github.com/shykes/gorp" //Forked to implement CreateTablesOpts - "io" - "io/ioutil" - "os" - "path" - "path/filepath" - "regexp" - "strings" - "syscall" - "time" -) - -type Store struct { - Root string - db *sql.DB - orm *gorp.DbMap - layers *LayerStore -} - -type Archive io.Reader - -func New(root string) (*Store, error) { - isNewStore := true - - if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { - return nil, err - } - db, err := sql.Open("sqlite3", path.Join(root, "db")) - if err != nil { - return nil, err - } - orm := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} - orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id") - orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") - orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root") - orm.AddTableWithName(Tag{}, "tags").SetKeys(false, "TagName") - if isNewStore { - if err := orm.CreateTablesOpts(true); err != nil { - return nil, err - } - } - - layers, err := NewLayerStore(path.Join(root, "layers")) - if err != nil { - return nil, err - } - return &Store{ - Root: root, - db: db, - orm: orm, - layers: layers, - }, nil -} - -func (store *Store) imageList(src []interface{}) []*Image { - var images []*Image - for _, i := range src { - img := i.(*Image) - img.store = store - images = append(images, img) - } - return images -} - -func (store *Store) Images() ([]*Image, error) { - images, err := store.orm.Select(Image{}, "select * from images") - if err != nil { - return nil, err - } - return store.imageList(images), nil -} - -func (store *Store) Paths() ([]string, error) { - var paths []string - rows, err := store.db.Query("select distinct Path from paths order by Path") - if err != nil { - return nil, err - } - for rows.Next() { - var path string - if err := rows.Scan(&path); err != nil { - return nil, err - } - paths = append(paths, path) - } - return paths, nil -} - -func (store *Store) RemoveInPath(pth string) error { - images, err := store.List(pth) - if err != nil { - return err - } - for _, img := range images { - if err = store.Remove(img); err != nil { - return err - } - } - return nil -} - -// DeleteMatch deletes all images whose name matches `pattern` -func (store *Store) RemoveRegexp(pattern string) error { - // Retrieve all the paths - paths, err := store.Paths() - if err != nil { - return err - } - // Check the pattern on each elements - for _, pth := range paths { - if match, err := regexp.MatchString(pattern, pth); err != nil { - return err - } else if match { - // If there is a match, remove it - if err := store.RemoveInPath(pth); err != nil { - return nil - } - } - } - return nil -} - -func (store *Store) Remove(img *Image) error { - _, err := store.orm.Delete(img) - return err -} - -func (store *Store) List(pth string) ([]*Image, error) { - pth = path.Clean(pth) - images, err := store.orm.Select(Image{}, "select images.* from images, paths where Path=? and paths.Image=images.Id order by images.Created desc", pth) - if err != nil { - return nil, err - } - return store.imageList(images), nil -} - -func (store *Store) Find(pth string) (*Image, error) { - pth = path.Clean(pth) - img, err := store.Get(pth) - if err != nil { - return nil, err - } else if img != nil { - return img, nil - } - - var q string - var args []interface{} - // FIXME: this breaks if the path contains a ':' - // If format is path:rev - if parts := strings.SplitN(pth, ":", 2); len(parts) == 2 { - q = "select Images.* from images, paths where Path=? and images.Id=? and paths.Image=images.Id" - args = []interface{}{parts[0], parts[1]} - // If format is path:rev - } else { - q = "select images.* from images, paths where Path=? and paths.Image=images.Id order by images.Created desc limit 1" - args = []interface{}{parts[0]} - } - images, err := store.orm.Select(Image{}, q, args...) - if err != nil { - return nil, err - } else if len(images) < 1 { - return nil, nil - } - img = images[0].(*Image) - img.store = store - return img, nil -} - -func (store *Store) Get(id string) (*Image, error) { - img, err := store.orm.Get(Image{}, id) - if img == nil { - return nil, err - } - res := img.(*Image) - res.store = store - return res, err -} - -func (store *Store) Create(layerData Archive, parent *Image, pth, comment string) (*Image, error) { - // FIXME: actually do something with the layer... - img := &Image{ - Id: future.RandomId(), - Comment: comment, - Created: time.Now().Unix(), - store: store, - } - if parent != nil { - img.Parent = parent.Id - } - // FIXME: Archive should contain compression info. For now we only support uncompressed. - err := store.Register(layerData, img, pth) - return img, err -} - -func (store *Store) Register(layerData Archive, img *Image, pth string) error { - img.store = store - _, err := store.layers.AddLayer(img.Id, layerData) - if err != nil { - return fmt.Errorf("Could not add layer: %s", err) - } - pathObj := &Path{ - Path: path.Clean(pth), - Image: img.Id, - } - trans, err := store.orm.Begin() - if err != nil { - return fmt.Errorf("Could not begin transaction: %s", err) - } - if err := trans.Insert(img); err != nil { - return fmt.Errorf("Could not insert image info: %s", err) - } - if err := trans.Insert(pathObj); err != nil { - return fmt.Errorf("Could not insert path info: %s", err) - } - if err := trans.Commit(); err != nil { - return fmt.Errorf("Could not commit transaction: %s", err) - } - return nil -} - -func (store *Store) Layers() []string { - return store.layers.List() -} - -type Image struct { - Id string - Parent string - Comment string - Created int64 - store *Store `db:"-"` -} - -func (image *Image) Copy(pth string) (*Image, error) { - if err := image.store.orm.Insert(&Path{Path: pth, Image: image.Id}); err != nil { - return nil, err - } - return image, nil -} - -type Mountpoint struct { - Image string - Root string - Rw string - Store *Store `db:"-"` -} - -func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { - mountpoint := &Mountpoint{ - Root: path.Clean(root), - Rw: path.Clean(rw), - Image: image.Id, - Store: image.store, - } - if err := image.store.orm.Insert(mountpoint); err != nil { - return nil, err - } - return mountpoint, nil -} - -func (image *Image) layers() ([]string, error) { - var list []string - var err error - currentImg := image - for currentImg != nil { - if layer := image.store.layers.Get(currentImg.Id); layer != "" { - list = append(list, layer) - } else { - return list, fmt.Errorf("Layer not found for image %s", image.Id) - } - currentImg, err = currentImg.store.Get(currentImg.Parent) - if err != nil { - return list, fmt.Errorf("Error while getting parent image: %v", err) - } - } - if len(list) == 0 { - return nil, fmt.Errorf("No layer found for image %s\n", image.Id) - } - return list, nil -} - -func (image *Image) Mountpoints() ([]*Mountpoint, error) { - var mountpoints []*Mountpoint - res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=?", image.Id) - if err != nil { - return nil, err - } - for _, mp := range res { - mountpoints = append(mountpoints, mp.(*Mountpoint)) - } - return mountpoints, nil -} - -func (image *Image) Mount(root, rw string) (*Mountpoint, error) { - var mountpoint *Mountpoint - if mp, err := image.store.FetchMountpoint(root, rw); err != nil { - return nil, err - } else if mp == nil { - mountpoint, err = image.Mountpoint(root, rw) - if err != nil { - return nil, fmt.Errorf("Could not create mountpoint: %s", err) - } else if mountpoint == nil { - return nil, fmt.Errorf("No mountpoint created") - } - } else { - mountpoint = mp - } - - if err := mountpoint.createFolders(); err != nil { - return nil, err - } - - // FIXME: Now mount the layers - rwBranch := fmt.Sprintf("%v=rw", mountpoint.Rw) - roBranches := "" - layers, err := image.layers() - if err != nil { - return nil, err - } - for _, layer := range layers { - roBranches += fmt.Sprintf("%v=ro:", layer) - } - branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) - if err := mount("none", mountpoint.Root, "aufs", 0, branches); err != nil { - return mountpoint, err - } - if !mountpoint.Mounted() { - return mountpoint, fmt.Errorf("Mount failed") - } - - // FIXME: Create tests for deletion - // FIXME: move this part to change.go, maybe refactor - // fs.Change() to avoid the fake mountpoint - // Retrieve the changeset from the parent and apply it to the container - // - Retrieve the changes - changes, err := image.store.Changes(&Mountpoint{ - Image: image.Id, - Root: layers[0], - Rw: layers[0], - Store: image.store}) - if err != nil { - return nil, err - } - // Iterate on changes - for _, c := range changes { - // If there is a delete - if c.Kind == ChangeDelete { - // Make sure the directory exists - file_path, file_name := path.Dir(c.Path), path.Base(c.Path) - if err := os.MkdirAll(path.Join(mountpoint.Rw, file_path), 0755); err != nil { - return nil, err - } - // And create the whiteout (we just need to create empty file, discard the return) - if _, err := os.Create(path.Join(path.Join(mountpoint.Rw, file_path), - ".wh."+path.Base(file_name))); err != nil { - return nil, err - } - } - } - return mountpoint, nil -} - -func (mp *Mountpoint) EnsureMounted() error { - if mp.Mounted() { - return nil - } - img, err := mp.Store.Get(mp.Image) - if err != nil { - return err - } - - _, err = img.Mount(mp.Root, mp.Rw) - return err -} - -func (mp *Mountpoint) createFolders() error { - if err := os.Mkdir(mp.Root, 0755); err != nil && !os.IsExist(err) { - return err - } - if err := os.Mkdir(mp.Rw, 0755); err != nil && !os.IsExist(err) { - return err - } - return nil -} - -func (mp *Mountpoint) Mounted() bool { - root, err := os.Stat(mp.Root) - if err != nil { - if os.IsNotExist(err) { - return false - } - panic(err) - } - parent, err := os.Stat(filepath.Join(mp.Root, "..")) - if err != nil { - panic(err) - } - - rootSt := root.Sys().(*syscall.Stat_t) - parentSt := parent.Sys().(*syscall.Stat_t) - return rootSt.Dev != parentSt.Dev -} - -func (mp *Mountpoint) Umount() error { - if !mp.Mounted() { - return fmt.Errorf("Mountpoint doesn't seem to be mounted") - } - if err := syscall.Unmount(mp.Root, 0); err != nil { - return fmt.Errorf("Unmount syscall failed: %v", err) - } - if mp.Mounted() { - return fmt.Errorf("Umount: Filesystem still mounted after calling umount(%v)", mp.Root) - } - // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint - // for some time. We'll just keep retrying until it succeeds. - for retries := 0; retries < 1000; retries++ { - err := os.Remove(mp.Root) - if err == nil { - // rm mntpoint succeeded - return nil - } - if os.IsNotExist(err) { - // mntpoint doesn't exist anymore. Success. - return nil - } - // fmt.Printf("(%v) Remove %v returned: %v\n", retries, mp.Root, err) - time.Sleep(10 * time.Millisecond) - } - return fmt.Errorf("Umount: Failed to umount %v", mp.Root) - -} - -func (mp *Mountpoint) Deregister() error { - if mp.Mounted() { - return fmt.Errorf("Mountpoint is currently mounted, can't deregister") - } - - _, err := mp.Store.orm.Delete(mp) - return err -} - -func (store *Store) FetchMountpoint(root, rw string) (*Mountpoint, error) { - res, err := store.orm.Select(Mountpoint{}, "select * from mountpoints where Root=? and Rw=?", root, rw) - if err != nil { - return nil, err - } else if len(res) < 1 || res[0] == nil { - return nil, nil - } - - mp := res[0].(*Mountpoint) - mp.Store = store - return mp, nil -} - -// OpenFile opens the named file for reading. -func (mp *Mountpoint) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { - if err := mp.EnsureMounted(); err != nil { - return nil, err - } - return os.OpenFile(filepath.Join(mp.Root, path), flag, perm) -} - -// ReadDir reads the directory named by dirname, relative to the Mountpoint's root, -// and returns a list of sorted directory entries -func (mp *Mountpoint) ReadDir(dirname string) ([]os.FileInfo, error) { - if err := mp.EnsureMounted(); err != nil { - return nil, err - } - return ioutil.ReadDir(filepath.Join(mp.Root, dirname)) -} - -func (store *Store) AddTag(imageId, tagName string) error { - if image, err := store.Get(imageId); err != nil { - return err - } else if image == nil { - return fmt.Errorf("No image with ID %s", imageId) - } - - err2 := store.orm.Insert(&Tag{ - TagName: tagName, - Image: imageId, - }) - - return err2 -} - -func (store *Store) GetByTag(tagName string) (*Image, error) { - res, err := store.orm.Get(Tag{}, tagName) - if err != nil { - return nil, err - } else if res == nil { - return nil, fmt.Errorf("No image associated to tag \"%s\"", tagName) - } - - tag := res.(*Tag) - - img, err2 := store.Get(tag.Image) - if err2 != nil { - return nil, err2 - } else if img == nil { - return nil, fmt.Errorf("Tag was found but image seems to be inexistent.") - } - - return img, nil -} - -type Path struct { - Path string - Image string -} - -type Tag struct { - TagName string - Image string -} diff --git a/fs/store_test.go b/fs/store_test.go deleted file mode 100644 index 8413ea0812..0000000000 --- a/fs/store_test.go +++ /dev/null @@ -1,277 +0,0 @@ -package fs - -import ( - "fmt" - "github.com/dotcloud/docker/future" - "io/ioutil" - "os" - "testing" - "time" -) - -func TestInit(t *testing.T) { - store, err := TempStore("testinit") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - paths, err := store.Paths() - if err != nil { - t.Fatal(err) - } - if l := len(paths); l != 0 { - t.Fatal("Fresh store should be empty after init (len=%d)", l) - } -} - -// FIXME: Do more extensive tests (ex: create multiple, delete, recreate; -// create multiple, check the amount of images and paths, etc..) -func TestCreate(t *testing.T) { - store, err := TempStore("testcreate") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - if images, err := store.Images(); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) - } - if images, err := store.List("foo"); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Path foo has wrong number of images (should be %d, not %d)", 1, l) - } else if images[0].Id != image.Id { - t.Fatalf("Imported image should be listed at path foo (%s != %s)", images[0], image) - } -} - -func TestRegister(t *testing.T) { - store, err := TempStore("testregister") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image := &Image{ - Id: future.RandomId(), - Comment: "testing", - Created: time.Now().Unix(), - store: store, - } - err = store.Register(archive, image, "foo") - if err != nil { - t.Fatal(err) - } - if images, err := store.Images(); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) - } - if images, err := store.List("foo"); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Path foo has wrong number of images (should be %d, not %d)", 1, l) - } else if images[0].Id != image.Id { - t.Fatalf("Imported image should be listed at path foo (%s != %s)", images[0], image) - } -} - -func TestTag(t *testing.T) { - store, err := TempStore("testtag") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - if images, err := store.Images(); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) - } - - if err := store.AddTag(image.Id, "baz"); err != nil { - t.Fatalf("Error while adding a tag to created image: %s", err) - } - - if taggedImage, err := store.GetByTag("baz"); err != nil { - t.Fatalf("Error while trying to retrieve image for tag 'baz': %s", err) - } else if taggedImage.Id != image.Id { - t.Fatalf("Expected to retrieve image %s but found %s instead", image.Id, taggedImage.Id) - } -} - -// Copy an image to a new path -func TestCopyNewPath(t *testing.T) { - store, err := TempStore("testcopynewpath") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - src, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - dst, err := src.Copy("bar") - if err != nil { - t.Fatal(err) - } - // ID should be the same - if src.Id != dst.Id { - t.Fatal("Different IDs") - } - // Check number of images at source path - if images, err := store.List("foo"); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatal("Wrong number of images at source path (should be %d, not %d)", 1, l) - } - // Check number of images at destination path - if images, err := store.List("bar"); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatal("Wrong number of images at destination path (should be %d, not %d)", 1, l) - } - if err := healthCheck(store); err != nil { - t.Fatal(err) - } -} - -// Copying an image to the same path twice should fail -func TestCopySameName(t *testing.T) { - store, err := TempStore("testcopysamename") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - src, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - _, err = src.Copy("foo") - if err == nil { - t.Fatal("Copying an image to the same patch twice should fail.") - } -} - -func TestMountPoint(t *testing.T) { - store, err := TempStore("test-mountpoint") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - mountpoint, err := image.Mountpoint("/tmp/a", "/tmp/b") - if err != nil { - t.Fatal(err) - } - if mountpoint.Root != "/tmp/a" { - t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/a", mountpoint.Root) - } - if mountpoint.Rw != "/tmp/b" { - t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/b", mountpoint.Rw) - } -} - -func TestMountpointDuplicateRoot(t *testing.T) { - store, err := TempStore("test-mountpoint") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - _, err = image.Mountpoint("/tmp/a", "/tmp/b") - if err != nil { - t.Fatal(err) - } - if _, err = image.Mountpoint("/tmp/a", "/tmp/foobar"); err == nil { - t.Fatal("Duplicate mountpoint root should fail") - } -} - -func TempStore(prefix string) (*Store, error) { - dir, err := ioutil.TempDir("", "docker-fs-test-"+prefix) - if err != nil { - return nil, err - } - return New(dir) -} - -func nuke(store *Store) error { - return os.RemoveAll(store.Root) -} - -// Look for inconsistencies in a store. -func healthCheck(store *Store) error { - parents := make(map[string]bool) - paths, err := store.Paths() - if err != nil { - return err - } - for _, path := range paths { - images, err := store.List(path) - if err != nil { - return err - } - IDs := make(map[string]bool) // All IDs for this path - for _, img := range images { - // Check for duplicate IDs per path - if _, exists := IDs[img.Id]; exists { - return fmt.Errorf("Duplicate ID: %s", img.Id) - } else { - IDs[img.Id] = true - } - // Store parent for 2nd pass - if parent := img.Parent; parent != "" { - parents[parent] = true - } - } - } - // Check non-existing parents - for parent := range parents { - if _, exists := parents[parent]; !exists { - return fmt.Errorf("Reference to non-registered parent: %s", parent) - } - } - return nil -} diff --git a/future/future.go b/future/future.go deleted file mode 100644 index 21b0eee385..0000000000 --- a/future/future.go +++ /dev/null @@ -1,136 +0,0 @@ -package future - -import ( - "bytes" - "crypto/sha256" - "errors" - "fmt" - "io" - "math/rand" - "net/http" - "time" -) - -func Seed() { - rand.Seed(time.Now().UTC().UnixNano()) -} - -func ComputeId(content io.Reader) (string, error) { - h := sha256.New() - if _, err := io.Copy(h, content); err != nil { - return "", err - } - return fmt.Sprintf("%x", h.Sum(nil)[:8]), nil -} - -func HumanDuration(d time.Duration) string { - if seconds := int(d.Seconds()); seconds < 1 { - return "Less than a second" - } else if seconds < 60 { - return fmt.Sprintf("%d seconds", seconds) - } else if minutes := int(d.Minutes()); minutes == 1 { - return "About a minute" - } else if minutes < 60 { - return fmt.Sprintf("%d minutes", minutes) - } else if hours := int(d.Hours()); hours == 1 { - return "About an hour" - } else if hours < 48 { - return fmt.Sprintf("%d hours", hours) - } else if hours < 24*7*2 { - return fmt.Sprintf("%d days", hours/24) - } else if hours < 24*30*3 { - return fmt.Sprintf("%d weeks", hours/24/7) - } else if hours < 24*365*2 { - return fmt.Sprintf("%d months", hours/24/30) - } - return fmt.Sprintf("%d years", d.Hours()/24/365) -} - -func randomBytes() io.Reader { - return bytes.NewBuffer([]byte(fmt.Sprintf("%x", rand.Int()))) -} - -func RandomId() string { - id, _ := ComputeId(randomBytes()) // can't fail - return id -} - -func Go(f func() error) chan error { - ch := make(chan error) - go func() { - ch <- f() - }() - return ch -} - -// Pv wraps an io.Reader such that it is passed through unchanged, -// but logs the number of bytes copied (comparable to the unix command pv) -func Pv(src io.Reader, info io.Writer) io.Reader { - var totalBytes int - data := make([]byte, 2048) - r, w := io.Pipe() - go func() { - for { - if n, err := src.Read(data); err != nil { - w.CloseWithError(err) - return - } else { - totalBytes += n - fmt.Fprintf(info, "--> %d bytes\n", totalBytes) - if _, err = w.Write(data[:n]); err != nil { - return - } - } - } - }() - return r -} - -// Request a given URL and return an io.Reader -func Download(url string, stderr io.Writer) (*http.Response, error) { - var resp *http.Response - var err error = nil - if resp, err = http.Get(url); err != nil { - return nil, err - } - if resp.StatusCode >= 400 { - return nil, errors.New("Got HTTP status code >= 400: " + resp.Status) - } - return resp, nil -} - -// Reader with progress bar -type progressReader struct { - reader io.ReadCloser // Stream to read from - output io.Writer // Where to send progress bar to - read_total int // Expected stream length (bytes) - read_progress int // How much has been read so far (bytes) - last_update int // How many bytes read at least update -} - -func (r *progressReader) Read(p []byte) (n int, err error) { - read, err := io.ReadCloser(r.reader).Read(p) - r.read_progress += read - - // Only update progress for every 1% read - update_every := int(0.01 * float64(r.read_total)) - if r.read_progress-r.last_update > update_every || r.read_progress == r.read_total { - fmt.Fprintf(r.output, "%d/%d (%.0f%%)\r", - r.read_progress, - r.read_total, - float64(r.read_progress)/float64(r.read_total)*100) - r.last_update = r.read_progress - } - // Send newline when complete - if err == io.EOF { - fmt.Fprintf(r.output, "\n") - } - - return read, err -} -func (r *progressReader) Close() error { - return io.ReadCloser(r.reader).Close() -} -func ProgressReader(r io.ReadCloser, size int, output io.Writer) *progressReader { - return &progressReader{r, output, size, 0, 0} -} diff --git a/graph.go b/graph.go new file mode 100644 index 0000000000..35f092703f --- /dev/null +++ b/graph.go @@ -0,0 +1,201 @@ +package docker + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "time" +) + +type Graph struct { + Root string +} + +func NewGraph(root string) (*Graph, error) { + abspath, err := filepath.Abs(root) + if err != nil { + return nil, err + } + // Create the root directory if it doesn't exists + if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + return nil, err + } + return &Graph{ + Root: abspath, + }, nil +} + +func (graph *Graph) Exists(id string) bool { + if _, err := graph.Get(id); err != nil { + return false + } + return true +} + +func (graph *Graph) Get(id string) (*Image, error) { + // FIXME: return nil when the image doesn't exist, instead of an error + img, err := LoadImage(graph.imageRoot(id)) + if err != nil { + return nil, err + } + if img.Id != id { + return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) + } + img.graph = graph + return img, nil +} + +func (graph *Graph) Create(layerData Archive, container *Container, comment string) (*Image, error) { + img := &Image{ + Id: GenerateId(), + Comment: comment, + Created: time.Now(), + } + if container != nil { + img.Parent = container.Image + img.Container = container.Id + img.ContainerConfig = *container.Config + } + if err := graph.Register(layerData, img); err != nil { + return nil, err + } + return img, nil +} + +func (graph *Graph) Register(layerData Archive, img *Image) error { + if err := ValidateId(img.Id); err != nil { + return err + } + // (This is a convenience to save time. Race conditions are taken care of by os.Rename) + if graph.Exists(img.Id) { + return fmt.Errorf("Image %s already exists", img.Id) + } + tmp, err := graph.Mktemp(img.Id) + defer os.RemoveAll(tmp) + if err != nil { + return fmt.Errorf("Mktemp failed: %s", err) + } + if err := StoreImage(img, layerData, tmp); err != nil { + return err + } + // Commit + if err := os.Rename(tmp, graph.imageRoot(img.Id)); err != nil { + return err + } + img.graph = graph + return nil +} + +func (graph *Graph) Mktemp(id string) (string, error) { + tmp, err := NewGraph(path.Join(graph.Root, ":tmp:")) + if err != nil { + return "", fmt.Errorf("Couldn't create temp: %s", err) + } + if tmp.Exists(id) { + return "", fmt.Errorf("Image %d already exists", id) + } + return tmp.imageRoot(id), nil +} + +func (graph *Graph) Garbage() (*Graph, error) { + return NewGraph(path.Join(graph.Root, ":garbage:")) +} + +func (graph *Graph) Delete(id string) error { + garbage, err := graph.Garbage() + if err != nil { + return err + } + return os.Rename(graph.imageRoot(id), garbage.imageRoot(id)) +} + +func (graph *Graph) Undelete(id string) error { + garbage, err := graph.Garbage() + if err != nil { + return err + } + return os.Rename(garbage.imageRoot(id), graph.imageRoot(id)) +} + +func (graph *Graph) GarbageCollect() error { + garbage, err := graph.Garbage() + if err != nil { + return err + } + return os.RemoveAll(garbage.Root) +} + +func (graph *Graph) Map() (map[string]*Image, error) { + // FIXME: this should replace All() + all, err := graph.All() + if err != nil { + return nil, err + } + images := make(map[string]*Image, len(all)) + for _, image := range all { + images[image.Id] = image + } + return images, nil +} + +func (graph *Graph) All() ([]*Image, error) { + var images []*Image + err := graph.WalkAll(func(image *Image) { + images = append(images, image) + }) + return images, err +} + +func (graph *Graph) WalkAll(handler func(*Image)) error { + files, err := ioutil.ReadDir(graph.Root) + if err != nil { + return err + } + for _, st := range files { + if img, err := graph.Get(st.Name()); err != nil { + // Skip image + continue + } else if handler != nil { + handler(img) + } + } + return nil +} + +func (graph *Graph) ByParent() (map[string][]*Image, error) { + byParent := make(map[string][]*Image) + err := graph.WalkAll(func(image *Image) { + image, err := graph.Get(image.Parent) + if err != nil { + return + } + if children, exists := byParent[image.Parent]; exists { + byParent[image.Parent] = []*Image{image} + } else { + byParent[image.Parent] = append(children, image) + } + }) + return byParent, err +} + +func (graph *Graph) Heads() (map[string]*Image, error) { + heads := make(map[string]*Image) + byParent, err := graph.ByParent() + if err != nil { + return nil, err + } + err = graph.WalkAll(func(image *Image) { + // If it's not in the byParent lookup table, then + // it's not a parent -> so it's a head! + if _, exists := byParent[image.Id]; !exists { + heads[image.Id] = image + } + }) + return heads, err +} + +func (graph *Graph) imageRoot(id string) string { + return path.Join(graph.Root, id) +} diff --git a/graph_test.go b/graph_test.go new file mode 100644 index 0000000000..b9bdc5140f --- /dev/null +++ b/graph_test.go @@ -0,0 +1,210 @@ +package docker + +import ( + "archive/tar" + "bytes" + "io" + "io/ioutil" + "os" + "path" + "testing" + "time" +) + +func TestInit(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + // Root should exist + if _, err := os.Stat(graph.Root); err != nil { + t.Fatal(err) + } + // All() should be empty + if l, err := graph.All(); err != nil { + t.Fatal(err) + } else if len(l) != 0 { + t.Fatalf("List() should return %d, not %d", 0, len(l)) + } +} + +// FIXME: Do more extensive tests (ex: create multiple, delete, recreate; +// create multiple, check the amount of images and paths, etc..) +func TestGraphCreate(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + archive, err := fakeTar() + if err != nil { + t.Fatal(err) + } + image, err := graph.Create(archive, nil, "Testing") + if err != nil { + t.Fatal(err) + } + if err := ValidateId(image.Id); err != nil { + t.Fatal(err) + } + if image.Comment != "Testing" { + t.Fatalf("Wrong comment: should be '%s', not '%s'", "Testing", image.Comment) + } + if images, err := graph.All(); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) + } +} + +func TestRegister(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + archive, err := fakeTar() + if err != nil { + t.Fatal(err) + } + image := &Image{ + Id: GenerateId(), + Comment: "testing", + Created: time.Now(), + } + err = graph.Register(archive, image) + if err != nil { + t.Fatal(err) + } + if images, err := graph.All(); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) + } + if resultImg, err := graph.Get(image.Id); err != nil { + t.Fatal(err) + } else { + if resultImg.Id != image.Id { + t.Fatalf("Wrong image ID. Should be '%s', not '%s'", image.Id, resultImg.Id) + } + if resultImg.Comment != image.Comment { + t.Fatalf("Wrong image comment. Should be '%s', not '%s'", image.Comment, resultImg.Comment) + } + } +} + +func TestMount(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + archive, err := fakeTar() + if err != nil { + t.Fatal(err) + } + image, err := graph.Create(archive, nil, "Testing") + if err != nil { + t.Fatal(err) + } + tmp, err := ioutil.TempDir("", "docker-test-graph-mount-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + rootfs := path.Join(tmp, "rootfs") + if err := os.MkdirAll(rootfs, 0700); err != nil { + t.Fatal(err) + } + rw := path.Join(tmp, "rw") + if err := os.MkdirAll(rw, 0700); err != nil { + t.Fatal(err) + } + if err := image.Mount(rootfs, rw); err != nil { + t.Fatal(err) + } + // FIXME: test for mount contents + defer func() { + if err := Unmount(rootfs); err != nil { + t.Error(err) + } + }() +} + +func TestDelete(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + archive, err := fakeTar() + if err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 0) + img, err := graph.Create(archive, nil, "Bla bla") + if err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 1) + if err := graph.Delete(img.Id); err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 0) + + // Test 2 create (same name) / 1 delete + img1, err := graph.Create(archive, nil, "Testing") + if err != nil { + t.Fatal(err) + } + if _, err = graph.Create(archive, nil, "Testing"); err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 2) + if err := graph.Delete(img1.Id); err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 1) + + // Test delete wrong name + if err := graph.Delete("Not_foo"); err == nil { + t.Fatalf("Deleting wrong ID should return an error") + } + assertNImages(graph, t, 1) + +} + +func assertNImages(graph *Graph, t *testing.T, n int) { + if images, err := graph.All(); err != nil { + t.Fatal(err) + } else if actualN := len(images); actualN != n { + t.Fatalf("Expected %d images, found %d", n, actualN) + } +} + +/* + * HELPER FUNCTIONS + */ + +func tempGraph(t *testing.T) *Graph { + tmp, err := ioutil.TempDir("", "docker-graph-") + if err != nil { + t.Fatal(err) + } + graph, err := NewGraph(tmp) + if err != nil { + t.Fatal(err) + } + return graph +} + +func testArchive(t *testing.T) Archive { + archive, err := fakeTar() + if err != nil { + t.Fatal(err) + } + return archive +} + +func fakeTar() (io.Reader, error) { + content := []byte("Hello world!\n") + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { + hdr := new(tar.Header) + hdr.Size = int64(len(content)) + hdr.Name = name + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + tw.Write([]byte(content)) + } + tw.Close() + return buf, nil +} diff --git a/image.go b/image.go new file mode 100644 index 0000000000..1a417a357b --- /dev/null +++ b/image.go @@ -0,0 +1,263 @@ +package docker + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "math/rand" + "os" + "path" + "strings" + "time" +) + +type Image struct { + Id string `json:"id"` + Parent string `json:"parent,omitempty"` + Comment string `json:"comment,omitempty"` + Created time.Time `json:"created"` + Container string `json:"container,omitempty"` + ContainerConfig Config `json:"container_config,omitempty"` + graph *Graph +} + +func LoadImage(root string) (*Image, error) { + // Load the json data + jsonData, err := ioutil.ReadFile(jsonPath(root)) + if err != nil { + return nil, err + } + var img Image + if err := json.Unmarshal(jsonData, &img); err != nil { + return nil, err + } + if err := ValidateId(img.Id); err != nil { + return nil, err + } + // Check that the filesystem layer exists + if stat, err := os.Stat(layerPath(root)); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("Couldn't load image %s: no filesystem layer", img.Id) + } else { + return nil, err + } + } else if !stat.IsDir() { + return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root)) + } + return &img, nil +} + +func StoreImage(img *Image, layerData Archive, root string) error { + // Check that root doesn't already exist + if _, err := os.Stat(root); err == nil { + return fmt.Errorf("Image %s already exists", img.Id) + } else if !os.IsNotExist(err) { + return err + } + // Store the layer + layer := layerPath(root) + if err := os.MkdirAll(layer, 0700); err != nil { + return err + } + if err := Untar(layerData, layer); err != nil { + return err + } + // Store the json ball + jsonData, err := json.Marshal(img) + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + return nil +} + +func layerPath(root string) string { + return path.Join(root, "layer") +} + +func jsonPath(root string) string { + return path.Join(root, "json") +} + +func MountAUFS(ro []string, rw string, target string) error { + // FIXME: Now mount the layers + rwBranch := fmt.Sprintf("%v=rw", rw) + roBranches := "" + for _, layer := range ro { + roBranches += fmt.Sprintf("%v=ro:", layer) + } + branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) + return mount("none", target, "aufs", 0, branches) +} + +func (image *Image) Mount(root, rw string) error { + if mounted, err := Mounted(root); err != nil { + return err + } else if mounted { + return fmt.Errorf("%s is already mounted", root) + } + layers, err := image.layers() + if err != nil { + return err + } + // Create the target directories if they don't exist + if err := os.Mkdir(root, 0755); err != nil && !os.IsExist(err) { + return err + } + if err := os.Mkdir(rw, 0755); err != nil && !os.IsExist(err) { + return err + } + // FIXME: @creack shouldn't we do this after going over changes? + if err := MountAUFS(layers, rw, root); err != nil { + return err + } + // FIXME: Create tests for deletion + // FIXME: move this part to change.go + // Retrieve the changeset from the parent and apply it to the container + // - Retrieve the changes + changes, err := Changes(layers, layers[0]) + if err != nil { + return err + } + // Iterate on changes + for _, c := range changes { + // If there is a delete + if c.Kind == ChangeDelete { + // Make sure the directory exists + file_path, file_name := path.Dir(c.Path), path.Base(c.Path) + if err := os.MkdirAll(path.Join(rw, file_path), 0755); err != nil { + return err + } + // And create the whiteout (we just need to create empty file, discard the return) + if _, err := os.Create(path.Join(path.Join(rw, file_path), + ".wh."+path.Base(file_name))); err != nil { + return err + } + } + } + return nil +} + +func (image *Image) Changes(rw string) ([]Change, error) { + layers, err := image.layers() + if err != nil { + return nil, err + } + return Changes(layers, rw) +} + +func ValidateId(id string) error { + if id == "" { + return fmt.Errorf("Image id can't be empty") + } + if strings.Contains(id, ":") { + return fmt.Errorf("Invalid character in image id: ':'") + } + return nil +} + +func GenerateId() string { + // FIXME: don't seed every time + rand.Seed(time.Now().UTC().UnixNano()) + randomBytes := bytes.NewBuffer([]byte(fmt.Sprintf("%x", rand.Int()))) + id, _ := ComputeId(randomBytes) // can't fail + return id +} + +// ComputeId reads from `content` until EOF, then returns a SHA of what it read, as a string. +func ComputeId(content io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, content); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)[:8]), nil +} + +// Image includes convenience proxy functions to its graph +// These functions will return an error if the image is not registered +// (ie. if image.graph == nil) +func (img *Image) History() ([]*Image, error) { + var parents []*Image + if err := img.WalkHistory( + func(img *Image) error { + parents = append(parents, img) + return nil + }, + ); err != nil { + return nil, err + } + return parents, nil +} + +// layers returns all the filesystem layers needed to mount an image +// FIXME: @shykes refactor this function with the new error handling +// (I'll do it if I have time tonight, I focus on the rest) +func (img *Image) layers() ([]string, error) { + var list []string + var e error + if err := img.WalkHistory( + func(img *Image) (err error) { + if layer, err := img.layer(); err != nil { + e = err + } else if layer != "" { + list = append(list, layer) + } + return err + }, + ); err != nil { + return nil, err + } else if e != nil { // Did an error occur inside the handler? + return nil, e + } + if len(list) == 0 { + return nil, fmt.Errorf("No layer found for image %s\n", img.Id) + } + return list, nil +} + +func (img *Image) WalkHistory(handler func(*Image) error) (err error) { + currentImg := img + for currentImg != nil { + if handler != nil { + if err := handler(currentImg); err != nil { + return err + } + } + currentImg, err = currentImg.GetParent() + if err != nil { + return fmt.Errorf("Error while getting parent image: %v", err) + } + } + return nil +} + +func (img *Image) GetParent() (*Image, error) { + if img.Parent == "" { + return nil, nil + } + if img.graph == nil { + return nil, fmt.Errorf("Can't lookup parent of unregistered image") + } + return img.graph.Get(img.Parent) +} + +func (img *Image) root() (string, error) { + if img.graph == nil { + return "", fmt.Errorf("Can't lookup root of unregistered image") + } + return img.graph.imageRoot(img.Id), nil +} + +// Return the path of an image's layer +func (img *Image) layer() (string, error) { + root, err := img.root() + if err != nil { + return "", err + } + return layerPath(root), nil +} diff --git a/lxc_template.go b/lxc_template.go index b86beb6bbc..e3beb037f9 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -22,7 +22,7 @@ lxc.network.mtu = 1500 lxc.network.ipv4 = {{.NetworkSettings.IpAddress}}/{{.NetworkSettings.IpPrefixLen}} # root filesystem -{{$ROOTFS := .Mountpoint.Root}} +{{$ROOTFS := .RootfsPath}} lxc.rootfs = {{$ROOTFS}} # use a dedicated pts for the container (and limit the number of pseudo terminal diff --git a/mount.go b/mount.go new file mode 100644 index 0000000000..bb1a40eddb --- /dev/null +++ b/mount.go @@ -0,0 +1,48 @@ +package docker + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "time" +) + +func Unmount(target string) error { + if err := syscall.Unmount(target, 0); err != nil { + return err + } + // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint + // for some time. We'll just keep retrying until it succeeds. + for retries := 0; retries < 1000; retries++ { + err := os.Remove(target) + if err == nil { + // rm mntpoint succeeded + return nil + } + if os.IsNotExist(err) { + // mntpoint doesn't exist anymore. Success. + return nil + } + // fmt.Printf("(%v) Remove %v returned: %v\n", retries, target, err) + time.Sleep(10 * time.Millisecond) + } + return fmt.Errorf("Umount: Failed to umount %v", target) +} + +func Mounted(mountpoint string) (bool, error) { + mntpoint, err := os.Stat(mountpoint) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + parent, err := os.Stat(filepath.Join(mountpoint, "..")) + if err != nil { + return false, err + } + mntpointSt := mntpoint.Sys().(*syscall.Stat_t) + parentSt := parent.Sys().(*syscall.Stat_t) + return mntpointSt.Dev != parentSt.Dev, nil +} diff --git a/mount_test.go b/mount_test.go deleted file mode 100644 index 56ac7eb417..0000000000 --- a/mount_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package docker - -import ( - "archive/tar" - "bytes" - "fmt" - "github.com/dotcloud/docker/fs" - "io" - "io/ioutil" - "os" - "testing" -) - -func fakeTar() (io.Reader, error) { - content := []byte("Hello world!\n") - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { - hdr := new(tar.Header) - hdr.Size = int64(len(content)) - hdr.Name = name - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - tw.Write([]byte(content)) - } - tw.Close() - return buf, nil -} - -// Look for inconsistencies in a store. -func healthCheck(store *fs.Store) error { - parents := make(map[string]bool) - paths, err := store.Paths() - if err != nil { - return err - } - for _, path := range paths { - images, err := store.List(path) - if err != nil { - return err - } - IDs := make(map[string]bool) // All IDs for this path - for _, img := range images { - // Check for duplicate IDs per path - if _, exists := IDs[img.Id]; exists { - return fmt.Errorf("Duplicate ID: %s", img.Id) - } else { - IDs[img.Id] = true - } - // Store parent for 2nd pass - if parent := img.Parent; parent != "" { - parents[parent] = true - } - } - } - // Check non-existing parents - for parent := range parents { - if _, exists := parents[parent]; !exists { - return fmt.Errorf("Reference to non-registered parent: %s", parent) - } - } - return nil -} - -// Note: This test is in the docker package because he needs to be run as root -func TestMount(t *testing.T) { - dir, err := ioutil.TempDir("", "docker-fs-test-mount") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - store, err := fs.New(dir) - if err != nil { - t.Fatal(err) - } - - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - - // Create mount targets - root, err := ioutil.TempDir("", "docker-fs-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(root) - - rw, err := ioutil.TempDir("", "docker-fs-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(rw) - - mountpoint, err := image.Mount(root, rw) - if err != nil { - t.Fatal(err) - } - defer mountpoint.Umount() - // Mountpoint should be marked as mounted - if !mountpoint.Mounted() { - t.Fatal("Mountpoint not mounted") - } - // There should be one mountpoint registered - if mps, err := image.Mountpoints(); err != nil { - t.Fatal(err) - } else if len(mps) != 1 { - t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 1, len(mps)) - } - // Unmounting should work - if err := mountpoint.Umount(); err != nil { - t.Fatal(err) - } - // De-registering should work - if err := mountpoint.Deregister(); err != nil { - t.Fatal(err) - } - if mps, err := image.Mountpoints(); err != nil { - t.Fatal(err) - } else if len(mps) != 0 { - t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 0, len(mps)) - } - // General health check - if err := healthCheck(store); err != nil { - t.Fatal(err) - } -} diff --git a/network_test.go b/network_test.go index c456b54838..251ec158fd 100644 --- a/network_test.go +++ b/network_test.go @@ -102,7 +102,7 @@ func TestConversion(t *testing.T) { func TestIPAllocator(t *testing.T) { gwIP, n, _ := net.ParseCIDR("127.0.0.1/29") - alloc, err := newIPAllocator(&net.IPNet{gwIP, n.Mask}) + alloc, err := newIPAllocator(&net.IPNet{IP: gwIP, Mask: n.Mask}) if err != nil { t.Fatal(err) } diff --git a/rcli/tcp.go b/rcli/tcp.go index 869a3bcdb6..a1fa669023 100644 --- a/rcli/tcp.go +++ b/rcli/tcp.go @@ -10,6 +10,11 @@ import ( "net" ) +// Note: the globals are here to avoid import cycle +// FIXME: Handle debug levels mode? +var DEBUG_FLAG bool = false +var CLIENT_SOCKET io.Writer = nil + // Connect to a remote endpoint using protocol `proto` and address `addr`, // issue a single call, and return the result. // `proto` may be "tcp", "unix", etc. See the `net` package for available protocols. @@ -42,6 +47,9 @@ func ListenAndServe(proto, addr string, service Service) error { return err } else { go func() { + if DEBUG_FLAG { + CLIENT_SOCKET = conn + } if err := Serve(conn, service); err != nil { log.Printf("Error: " + err.Error() + "\n") fmt.Fprintf(conn, "Error: "+err.Error()+"\n") diff --git a/registry.go b/registry.go new file mode 100644 index 0000000000..62394d63fc --- /dev/null +++ b/registry.go @@ -0,0 +1,385 @@ +package docker + +import ( + "encoding/json" + "fmt" + "github.com/dotcloud/docker/auth" + "io" + "io/ioutil" + "net/http" + "path" + "strings" +) + +//FIXME: Set the endpoint in a conf file or via commandline +//const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1" +const REGISTRY_ENDPOINT = auth.REGISTRY_SERVER + "/v1" + +// Build an Image object from raw json data +func NewImgJson(src []byte) (*Image, error) { + ret := &Image{} + + Debugf("Json string: {%s}\n", src) + // FIXME: Is there a cleaner way to "puryfy" the input json? + if err := json.Unmarshal(src, ret); err != nil { + return nil, err + } + return ret, nil +} + +// Build an Image object list from a raw json data +// FIXME: Do this in "stream" mode +func NewMultipleImgJson(src []byte) ([]*Image, error) { + ret := []*Image{} + + dec := json.NewDecoder(strings.NewReader(strings.Replace(string(src), "null", "\"\"", -1))) + for { + m := &Image{} + if err := dec.Decode(m); err == io.EOF { + break + } else if err != nil { + return nil, err + } + ret = append(ret, m) + } + return ret, nil +} + +// Retrieve the history of a given image from the Registry. +// Return a list of the parent's json (requested image included) +func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) ([]*Image, error) { + client := &http.Client{} + + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/history", nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + if res != nil { + return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId) + } + return nil, err + } + defer res.Body.Close() + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s\n", err) + } + + history, err := NewMultipleImgJson(jsonString) + if err != nil { + return nil, fmt.Errorf("Error while parsing the json: %s\n", err) + } + return history, nil +} + +// Check if an image exists in the Registry +func (graph *Graph) LookupRemoteImage(imgId string, authConfig *auth.AuthConfig) bool { + rt := &http.Transport{Proxy: http.ProxyFromEnvironment} + + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) + if err != nil { + return false + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := rt.RoundTrip(req) + if err != nil || res.StatusCode != 307 { + return false + } + return res.StatusCode == 307 +} + +// Retrieve an image from the Registry. +// Returns the Image object as well as the layer as an Archive (io.Reader) +func (graph *Graph) getRemoteImage(imgId string, authConfig *auth.AuthConfig) (*Image, Archive, error) { + client := &http.Client{} + + // Get the Json + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) + if err != nil { + return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + if res != nil { + return nil, nil, fmt.Errorf("Internal server error: %d trying to get image %s", res.StatusCode, imgId) + } + return nil, nil, err + } + defer res.Body.Close() + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, nil, fmt.Errorf("Error while reading the http response: %s\n", err) + } + + img, err := NewImgJson(jsonString) + if err != nil { + return nil, nil, fmt.Errorf("Error while parsing the json: %s\n", err) + } + img.Id = imgId + + // Get the layer + req, err = http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/layer", nil) + if err != nil { + return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err = client.Do(req) + if err != nil { + return nil, nil, err + } + return img, res.Body, nil +} + +func (graph *Graph) PullImage(imgId string, authConfig *auth.AuthConfig) error { + history, err := graph.getRemoteHistory(imgId, authConfig) + if err != nil { + return err + } + // FIXME: Try to stream the images? + // FIXME: Lunch the getRemoteImage() in goroutines + for _, j := range history { + if !graph.Exists(j.Id) { + img, layer, err := graph.getRemoteImage(j.Id, authConfig) + if err != nil { + // FIXME: Keep goging in case of error? + return err + } + if err = graph.Register(layer, img); err != nil { + return err + } + } + } + return nil +} + +// FIXME: Handle the askedTag parameter +func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { + client := &http.Client{} + + fmt.Fprintf(stdout, "Pulling repo: %s\n", REGISTRY_ENDPOINT+"/users/"+remote) + + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/users/"+remote, nil) + if err != nil { + return err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + if res != nil { + return fmt.Errorf("Internal server error: %d trying to pull %s", res.StatusCode, remote) + } + return err + } + defer res.Body.Close() + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + t := map[string]string{} + if err = json.Unmarshal(rawJson, &t); err != nil { + return err + } + for tag, rev := range t { + if err = graph.PullImage(rev, authConfig); err != nil { + return err + } + if err = repositories.Set(remote, tag, rev, true); err != nil { + return err + } + } + if err = repositories.Save(); err != nil { + return err + } + return nil +} + +// Push a local image to the registry with its history if needed +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth.AuthConfig) error { + client := &http.Client{} + + // FIXME: Factorize the code + // FIXME: Do the puts in goroutines + if err := imgOrig.WalkHistory(func(img *Image) error { + + jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) + if err != nil { + return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) + } + + fmt.Fprintf(stdout, "Pushing image [%s] on {%s}\n", img.Id, REGISTRY_ENDPOINT+"/images/"+img.Id+"/json") + + // FIXME: try json with UTF8 + jsonData := strings.NewReader(string(jsonRaw)) + req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData) + if err != nil { + return err + } + req.Header.Add("Content-type", "application/json") + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + if res == nil { + return fmt.Errorf( + "Error: Internal server error trying to push image {%s} (json): %s", + img.Id, err) + } + Debugf("Pushing return status: %d\n", res.StatusCode) + switch res.StatusCode { + case 204: + // Case where the image is already on the Registry + // FIXME: Do not be silent? + fmt.Fprintf(stdout, "The image %s is already up to date on the registry.\n", img.Id) + return nil + case 400: + return fmt.Errorf("Error: Invalid Json") + default: + return fmt.Errorf( + "Error: Internal server error: %d trying to push image {%s} (json): %s\n", + res.StatusCode, img.Id, err) + } + } + + req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil) + req2.SetBasicAuth(authConfig.Username, authConfig.Password) + res2, err := client.Do(req2) + if err != nil || res2.StatusCode != 307 { + return fmt.Errorf( + "Internal server error trying to push image {%s} (layer 1): %s\n", + img.Id, err) + } + url, err := res2.Location() + if err != nil || url == nil { + return fmt.Errorf( + "Fail to retrieve layer storage URL for image {%s}: %s\n", + img.Id, err) + } + + // FIXME: Don't do this :D. Check the S3 requierement and implement chunks of 5MB + // FIXME2: I won't stress it enough, DON'T DO THIS! very high priority + layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Gzip) + layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Gzip) + if err != nil { + return fmt.Errorf( + "Error while retrieving layer for {%s}: %s\n", + img.Id, err) + } + req3, err := http.NewRequest("PUT", url.String(), layerData) + if err != nil { + return err + } + tmp, err := ioutil.ReadAll(layerData2) + if err != nil { + return err + } + req3.ContentLength = int64(len(tmp)) + + req3.TransferEncoding = []string{"none"} + res3, err := client.Do(req3) + if err != nil || res3.StatusCode != 200 { + if res3 == nil { + return fmt.Errorf( + "Error trying to push image {%s} (layer 2): %s\n", + img.Id, err) + } + return fmt.Errorf( + "Error trying to push image {%s} (layer 2): %s (%d)\n", + img.Id, err, res3.StatusCode) + } + return nil + }); err != nil { + return err + } + return nil +} + +// push a tag on the registry. +// Remote has the format '/ +func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthConfig) error { + + // Keep this for backward compatibility + if tag == "" { + tag = "lastest" + } + + // "jsonify" the string + revision = "\"" + revision + "\"" + + Debugf("Pushing tags for rev [%s] on {%s}\n", revision, REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag) + + client := &http.Client{} + req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag, strings.NewReader(revision)) + req.Header.Add("Content-type", "application/json") + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) + if err != nil || (res.StatusCode != 200 && res.StatusCode != 201) { + if res != nil { + return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) + } + return err + } + Debugf("Result of push tag: %d\n", res.StatusCode) + switch res.StatusCode { + default: + return fmt.Errorf("Error %d\n", res.StatusCode) + case 200: + case 201: + } + return nil +} + +func (graph *Graph) LookupRemoteRepository(remote string, authConfig *auth.AuthConfig) bool { + rt := &http.Transport{Proxy: http.ProxyFromEnvironment} + + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/users/"+remote, nil) + if err != nil { + return false + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := rt.RoundTrip(req) + if err != nil || res.StatusCode != 200 { + return false + } + return true +} + +func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, authConfig *auth.AuthConfig) error { + // CHeck if the local impage exists + img, err := graph.Get(imgId) + if err != nil { + return err + } + // Push the image + if err = graph.PushImage(stdout, img, authConfig); err != nil { + return err + } + // And then the tag + if err = graph.pushTag(remote, imgId, tag, authConfig); err != nil { + return err + } + return nil +} + +// Push a repository to the registry. +// Remote has the format '/ +func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { + // Check if the remote repository exists + // FIXME: @lopter How to handle this? + // if !graph.LookupRemoteRepository(remote, authConfig) { + // return fmt.Errorf("The remote repository %s does not exist\n", remote) + // } + + // For each image within the repo, push them + for tag, imgId := range localRepo { + if err := graph.pushPrimitive(stdout, remote, tag, imgId, authConfig); err != nil { + // FIXME: Continue on error? + return err + } + } + return nil +} diff --git a/runtime.go b/runtime.go new file mode 100644 index 0000000000..b7f2582b46 --- /dev/null +++ b/runtime.go @@ -0,0 +1,292 @@ +package docker + +import ( + "container/list" + "fmt" + "github.com/dotcloud/docker/auth" + "io" + "io/ioutil" + "os" + "path" + "sort" + "sync" + "time" +) + +type Runtime struct { + root string + repository string + containers *list.List + networkManager *NetworkManager + graph *Graph + repositories *TagStore + authConfig *auth.AuthConfig +} + +var sysInitPath string + +func init() { + sysInitPath = SelfPath() +} + +func (runtime *Runtime) List() []*Container { + containers := new(History) + for e := runtime.containers.Front(); e != nil; e = e.Next() { + containers.Add(e.Value.(*Container)) + } + return *containers +} + +func (runtime *Runtime) getContainerElement(id string) *list.Element { + for e := runtime.containers.Front(); e != nil; e = e.Next() { + container := e.Value.(*Container) + if container.Id == id { + return e + } + } + return nil +} + +func (runtime *Runtime) Get(id string) *Container { + e := runtime.getContainerElement(id) + if e == nil { + return nil + } + return e.Value.(*Container) +} + +func (runtime *Runtime) Exists(id string) bool { + return runtime.Get(id) != nil +} + +func (runtime *Runtime) containerRoot(id string) string { + return path.Join(runtime.repository, id) +} + +func (runtime *Runtime) Create(config *Config) (*Container, error) { + // Lookup image + img, err := runtime.repositories.LookupImage(config.Image) + if err != nil { + return nil, err + } + container := &Container{ + // FIXME: we should generate the ID here instead of receiving it as an argument + Id: GenerateId(), + Created: time.Now(), + Path: config.Cmd[0], + Args: config.Cmd[1:], //FIXME: de-duplicate from config + Config: config, + Image: img.Id, // Always use the resolved image id + NetworkSettings: &NetworkSettings{}, + // FIXME: do we need to store this in the container? + SysInitPath: sysInitPath, + } + container.root = runtime.containerRoot(container.Id) + // Step 1: create the container directory. + // This doubles as a barrier to avoid race conditions. + if err := os.Mkdir(container.root, 0700); err != nil { + return nil, err + } + // Step 2: save the container json + if err := container.ToDisk(); err != nil { + return nil, err + } + // Step 3: register the container + if err := runtime.Register(container); err != nil { + return nil, err + } + return container, nil +} + +func (runtime *Runtime) Load(id string) (*Container, error) { + container := &Container{root: runtime.containerRoot(id)} + if err := container.FromDisk(); err != nil { + return nil, err + } + if container.Id != id { + return container, fmt.Errorf("Container %s is stored at %s", container.Id, id) + } + if err := runtime.Register(container); err != nil { + return nil, err + } + return container, nil +} + +// Register makes a container object usable by the runtime as +func (runtime *Runtime) Register(container *Container) error { + if container.runtime != nil || runtime.Exists(container.Id) { + return fmt.Errorf("Container is already loaded") + } + if err := validateId(container.Id); err != nil { + return err + } + container.runtime = runtime + // Setup state lock (formerly in newState() + lock := new(sync.Mutex) + container.State.stateChangeLock = lock + container.State.stateChangeCond = sync.NewCond(lock) + // Attach to stdout and stderr + container.stderr = newWriteBroadcaster() + container.stdout = newWriteBroadcaster() + // Attach to stdin + if container.Config.OpenStdin { + container.stdin, container.stdinPipe = io.Pipe() + } else { + container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin + } + // Setup logging of stdout and stderr to disk + if err := runtime.LogToDisk(container.stdout, container.logPath("stdout")); err != nil { + return err + } + if err := runtime.LogToDisk(container.stderr, container.logPath("stderr")); err != nil { + return err + } + // done + runtime.containers.PushBack(container) + return nil +} + +func (runtime *Runtime) LogToDisk(src *writeBroadcaster, dst string) error { + log, err := os.OpenFile(dst, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600) + if err != nil { + return err + } + src.AddWriter(NopWriteCloser(log)) + return nil +} + +func (runtime *Runtime) Destroy(container *Container) error { + element := runtime.getContainerElement(container.Id) + if element == nil { + return fmt.Errorf("Container %v not found - maybe it was already destroyed?", container.Id) + } + + if err := container.Stop(); err != nil { + return err + } + if mounted, err := container.Mounted(); err != nil { + return err + } else if mounted { + if err := container.Unmount(); err != nil { + return fmt.Errorf("Unable to unmount container %v: %v", container.Id, err) + } + } + // Deregister the container before removing its directory, to avoid race conditions + runtime.containers.Remove(element) + if err := os.RemoveAll(container.root); err != nil { + return fmt.Errorf("Unable to remove filesystem for %v: %v", container.Id, err) + } + return nil +} + +// Commit creates a new filesystem image from the current state of a container. +// The image can optionally be tagged into a repository +func (runtime *Runtime) Commit(id, repository, tag string) (*Image, error) { + container := runtime.Get(id) + if container == nil { + return nil, fmt.Errorf("No such container: %s", id) + } + // FIXME: freeze the container before copying it to avoid data corruption? + // FIXME: this shouldn't be in commands. + rwTar, err := container.ExportRw() + if err != nil { + return nil, err + } + // Create a new image from the container's base layers + a new layer from container changes + img, err := runtime.graph.Create(rwTar, container, "") + if err != nil { + return nil, err + } + // Register the image if needed + if repository != "" { + if err := runtime.repositories.Set(repository, tag, img.Id, true); err != nil { + return img, err + } + } + return img, nil +} + +func (runtime *Runtime) restore() error { + dir, err := ioutil.ReadDir(runtime.repository) + if err != nil { + return err + } + for _, v := range dir { + id := v.Name() + container, err := runtime.Load(id) + if err != nil { + Debugf("Failed to load container %v: %v", id, err) + continue + } + Debugf("Loaded container %v", container.Id) + } + return nil +} + +func NewRuntime() (*Runtime, error) { + return NewRuntimeFromDirectory("/var/lib/docker") +} + +func NewRuntimeFromDirectory(root string) (*Runtime, error) { + runtime_repo := path.Join(root, "containers") + + if err := os.MkdirAll(runtime_repo, 0700); err != nil && !os.IsExist(err) { + return nil, err + } + + g, err := NewGraph(path.Join(root, "graph")) + if err != nil { + return nil, err + } + repositories, err := NewTagStore(path.Join(root, "repositories"), g) + if err != nil { + return nil, fmt.Errorf("Couldn't create Tag store: %s", err) + } + netManager, err := newNetworkManager(networkBridgeIface) + if err != nil { + return nil, err + } + authConfig, err := auth.LoadConfig(root) + if err != nil && authConfig == nil { + // If the auth file does not exist, keep going + return nil, err + } + + runtime := &Runtime{ + root: root, + repository: runtime_repo, + containers: list.New(), + networkManager: netManager, + graph: g, + repositories: repositories, + authConfig: authConfig, + } + + if err := runtime.restore(); err != nil { + return nil, err + } + return runtime, nil +} + +type History []*Container + +func (history *History) Len() int { + return len(*history) +} + +func (history *History) Less(i, j int) bool { + containers := *history + return containers[j].When().Before(containers[i].When()) +} + +func (history *History) Swap(i, j int) { + containers := *history + tmp := containers[i] + containers[i] = containers[j] + containers[j] = tmp +} + +func (history *History) Add(container *Container) { + *history = append(*history, container) + sort.Sort(history) +} diff --git a/docker_test.go b/runtime_test.go similarity index 52% rename from docker_test.go rename to runtime_test.go index 735e15baf7..76882f95a9 100644 --- a/docker_test.go +++ b/runtime_test.go @@ -1,7 +1,6 @@ package docker import ( - "github.com/dotcloud/docker/fs" "io" "io/ioutil" "os" @@ -11,13 +10,13 @@ import ( ) const testLayerPath string = "/var/lib/docker/docker-ut.tar" -const unitTestImageName string = "busybox" +const unitTestImageName string = "http://get.docker.io/images/busybox" var unitTestStoreBase string var srv *Server -func nuke(docker *Docker) error { - return os.RemoveAll(docker.root) +func nuke(runtime *Runtime) error { + return os.RemoveAll(runtime.root) } func CopyDirectory(source, dest string) error { @@ -57,14 +56,13 @@ func init() { unitTestStoreBase = root // Make it our Store root - docker, err := NewFromDirectory(root) + runtime, err := NewRuntimeFromDirectory(root) if err != nil { panic(err) } // Create the "Server" srv := &Server{ - images: docker.Store, - containers: docker, + runtime: runtime, } // Retrieve the Image if err := srv.CmdImport(os.Stdin, os.Stdout, unitTestImageName); err != nil { @@ -72,7 +70,7 @@ func init() { } } -func newTestDocker() (*Docker, error) { +func newTestRuntime() (*Runtime, error) { root, err := ioutil.TempDir("", "docker-test") if err != nil { return nil, err @@ -85,16 +83,16 @@ func newTestDocker() (*Docker, error) { return nil, err } - docker, err := NewFromDirectory(root) + runtime, err := NewRuntimeFromDirectory(root) if err != nil { return nil, err } - return docker, nil + return runtime, nil } -func GetTestImage(docker *Docker) *fs.Image { - imgs, err := docker.Store.Images() +func GetTestImage(runtime *Runtime) *Image { + imgs, err := runtime.graph.All() if err != nil { panic(err) } else if len(imgs) < 1 { @@ -103,161 +101,151 @@ func GetTestImage(docker *Docker) *fs.Image { return imgs[0] } -func TestCreate(t *testing.T) { - docker, err := newTestDocker() +func TestRuntimeCreate(t *testing.T) { + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) // Make sure we start we 0 containers - if len(docker.List()) != 0 { - t.Errorf("Expected 0 containers, %v found", len(docker.List())) + if len(runtime.List()) != 0 { + t.Errorf("Expected 0 containers, %v found", len(runtime.List())) } - container, err := docker.Create( - "test_create", - "ls", - []string{"-al"}, - GetTestImage(docker), - &Config{}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) } defer func() { - if err := docker.Destroy(container); err != nil { + if err := runtime.Destroy(container); err != nil { t.Error(err) } }() // Make sure we can find the newly created container with List() - if len(docker.List()) != 1 { - t.Errorf("Expected 1 container, %v found", len(docker.List())) + if len(runtime.List()) != 1 { + t.Errorf("Expected 1 container, %v found", len(runtime.List())) } // Make sure the container List() returns is the right one - if docker.List()[0].Id != "test_create" { - t.Errorf("Unexpected container %v returned by List", docker.List()[0]) + if runtime.List()[0].Id != container.Id { + t.Errorf("Unexpected container %v returned by List", runtime.List()[0]) } // Make sure we can get the container with Get() - if docker.Get("test_create") == nil { + if runtime.Get(container.Id) == nil { t.Errorf("Unable to get newly created container") } // Make sure it is the right container - if docker.Get("test_create") != container { + if runtime.Get(container.Id) != container { t.Errorf("Get() returned the wrong container") } // Make sure Exists returns it as existing - if !docker.Exists("test_create") { + if !runtime.Exists(container.Id) { t.Errorf("Exists() returned false for a newly created container") } } func TestDestroy(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( - "test_destroy", - "ls", - []string{"-al"}, - GetTestImage(docker), - &Config{}, + defer nuke(runtime) + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) } // Destroy - if err := docker.Destroy(container); err != nil { + if err := runtime.Destroy(container); err != nil { t.Error(err) } - // Make sure docker.Exists() behaves correctly - if docker.Exists("test_destroy") { + // Make sure runtime.Exists() behaves correctly + if runtime.Exists("test_destroy") { t.Errorf("Exists() returned true") } - // Make sure docker.List() doesn't list the destroyed container - if len(docker.List()) != 0 { - t.Errorf("Expected 0 container, %v found", len(docker.List())) + // Make sure runtime.List() doesn't list the destroyed container + if len(runtime.List()) != 0 { + t.Errorf("Expected 0 container, %v found", len(runtime.List())) } - // Make sure docker.Get() refuses to return the unexisting container - if docker.Get("test_destroy") != nil { + // Make sure runtime.Get() refuses to return the unexisting container + if runtime.Get(container.Id) != nil { t.Errorf("Unable to get newly created container") } // Make sure the container root directory does not exist anymore - _, err = os.Stat(container.Root) + _, err = os.Stat(container.root) if err == nil || !os.IsNotExist(err) { t.Errorf("Container root directory still exists after destroy") } // Test double destroy - if err := docker.Destroy(container); err == nil { + if err := runtime.Destroy(container); err == nil { // It should have failed t.Errorf("Double destroy did not fail") } } func TestGet(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container1, err := docker.Create( - "test1", - "ls", - []string{"-al"}, - GetTestImage(docker), - &Config{}, + defer nuke(runtime) + container1, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container1) + defer runtime.Destroy(container1) - container2, err := docker.Create( - "test2", - "ls", - []string{"-al"}, - GetTestImage(docker), - &Config{}, + container2, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container2) + defer runtime.Destroy(container2) - container3, err := docker.Create( - "test3", - "ls", - []string{"-al"}, - GetTestImage(docker), - &Config{}, + container3, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container3) + defer runtime.Destroy(container3) - if docker.Get("test1") != container1 { - t.Errorf("Get(test1) returned %v while expecting %v", docker.Get("test1"), container1) + if runtime.Get(container1.Id) != container1 { + t.Errorf("Get(test1) returned %v while expecting %v", runtime.Get(container1.Id), container1) } - if docker.Get("test2") != container2 { - t.Errorf("Get(test2) returned %v while expecting %v", docker.Get("test2"), container2) + if runtime.Get(container2.Id) != container2 { + t.Errorf("Get(test2) returned %v while expecting %v", runtime.Get(container2.Id), container2) } - if docker.Get("test3") != container3 { - t.Errorf("Get(test3) returned %v while expecting %v", docker.Get("test3"), container3) + if runtime.Get(container3.Id) != container3 { + t.Errorf("Get(test3) returned %v while expecting %v", runtime.Get(container3.Id), container3) } } @@ -275,25 +263,23 @@ func TestRestore(t *testing.T) { t.Fatal(err) } - docker1, err := NewFromDirectory(root) + runtime1, err := NewRuntimeFromDirectory(root) if err != nil { t.Fatal(err) } // Create a container with one instance of docker - container1, err := docker1.Create( - "restore_test", - "ls", - []string{"-al"}, - GetTestImage(docker1), - &Config{}, + container1, err := runtime1.Create(&Config{ + Image: GetTestImage(runtime1).Id, + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) } - defer docker1.Destroy(container1) - if len(docker1.List()) != 1 { - t.Errorf("Expected 1 container, %v found", len(docker1.List())) + defer runtime1.Destroy(container1) + if len(runtime1.List()) != 1 { + t.Errorf("Expected 1 container, %v found", len(runtime1.List())) } if err := container1.Run(); err != nil { t.Fatal(err) @@ -301,15 +287,15 @@ func TestRestore(t *testing.T) { // Here are are simulating a docker restart - that is, reloading all containers // from scratch - docker2, err := NewFromDirectory(root) + runtime2, err := NewRuntimeFromDirectory(root) if err != nil { t.Fatal(err) } - defer nuke(docker2) - if len(docker2.List()) != 1 { - t.Errorf("Expected 1 container, %v found", len(docker2.List())) + defer nuke(runtime2) + if len(runtime2.List()) != 1 { + t.Errorf("Expected 1 container, %v found", len(runtime2.List())) } - container2 := docker2.Get("restore_test") + container2 := runtime2.Get(container1.Id) if container2 == nil { t.Fatal("Unable to Get container") } diff --git a/state.go b/state.go index e864f304f9..f438ff8727 100644 --- a/state.go +++ b/state.go @@ -2,7 +2,6 @@ package docker import ( "fmt" - "github.com/dotcloud/docker/future" "sync" "time" ) @@ -17,18 +16,10 @@ type State struct { stateChangeCond *sync.Cond } -func newState() *State { - lock := new(sync.Mutex) - return &State{ - stateChangeLock: lock, - stateChangeCond: sync.NewCond(lock), - } -} - // String returns a human-readable description of the state func (s *State) String() string { if s.Running { - return fmt.Sprintf("Up %s", future.HumanDuration(time.Now().Sub(s.StartedAt))) + return fmt.Sprintf("Up %s", HumanDuration(time.Now().Sub(s.StartedAt))) } return fmt.Sprintf("Exit %d", s.ExitCode) } diff --git a/sysinit.go b/sysinit.go index c475b3365d..f701417978 100644 --- a/sysinit.go +++ b/sysinit.go @@ -52,13 +52,6 @@ func changeUser(u string) { } } -// Set the environment to a known, repeatable state -func setupEnv() { - os.Clearenv() - os.Setenv("HOME", "/") - os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") -} - func executeProgram(name string, args []string) { path, err := exec.LookPath(name) if err != nil { @@ -86,6 +79,5 @@ func SysInit() { setupNetworking(*gw) changeUser(*u) - setupEnv() executeProgram(flag.Arg(0), flag.Args()) } diff --git a/tags.go b/tags.go new file mode 100644 index 0000000000..1df7c6e463 --- /dev/null +++ b/tags.go @@ -0,0 +1,184 @@ +package docker + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const DEFAULT_TAG = "latest" + +type TagStore struct { + path string + graph *Graph + Repositories map[string]Repository +} + +type Repository map[string]string + +func NewTagStore(path string, graph *Graph) (*TagStore, error) { + abspath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + store := &TagStore{ + path: abspath, + graph: graph, + Repositories: make(map[string]Repository), + } + // Load the json file if it exists, otherwise create it. + if err := store.Reload(); os.IsNotExist(err) { + if err := store.Save(); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + return store, nil +} + +func (store *TagStore) Save() error { + // Store the json ball + jsonData, err := json.Marshal(store) + if err != nil { + return err + } + if err := ioutil.WriteFile(store.path, jsonData, 0600); err != nil { + return err + } + return nil +} + +func (store *TagStore) Reload() error { + jsonData, err := ioutil.ReadFile(store.path) + if err != nil { + return err + } + if err := json.Unmarshal(jsonData, store); err != nil { + return err + } + return nil +} + +func (store *TagStore) LookupImage(name string) (*Image, error) { + img, err := store.graph.Get(name) + if err != nil { + // FIXME: standardize on returning nil when the image doesn't exist, and err for everything else + // (so we can pass all errors here) + repoAndTag := strings.SplitN(name, ":", 2) + if len(repoAndTag) == 1 { + repoAndTag = append(repoAndTag, DEFAULT_TAG) + } + if i, err := store.GetImage(repoAndTag[0], repoAndTag[1]); err != nil { + return nil, err + } else if i == nil { + return nil, fmt.Errorf("No such image: %s", name) + } else { + img = i + } + } + return img, nil +} + +// Return a reverse-lookup table of all the names which refer to each image +// Eg. {"43b5f19b10584": {"base:latest", "base:v1"}} +func (store *TagStore) ById() map[string][]string { + byId := make(map[string][]string) + for repoName, repository := range store.Repositories { + for tag, id := range repository { + name := repoName + ":" + tag + if _, exists := byId[id]; !exists { + byId[id] = []string{name} + } else { + byId[id] = append(byId[id], name) + } + } + } + return byId +} + +func (store *TagStore) ImageName(id string) string { + if names, exists := store.ById()[id]; exists && len(names) > 0 { + return names[0] + } + return id +} + +func (store *TagStore) Set(repoName, tag, imageName string, force bool) error { + img, err := store.LookupImage(imageName) + if err != nil { + return err + } + if tag == "" { + tag = DEFAULT_TAG + } + if err := validateRepoName(repoName); err != nil { + return err + } + if err := validateTagName(tag); err != nil { + return err + } + if err := store.Reload(); err != nil { + return err + } + var repo Repository + if r, exists := store.Repositories[repoName]; exists { + repo = r + } else { + repo = make(map[string]string) + if old, exists := store.Repositories[repoName]; exists && !force { + return fmt.Errorf("Tag %s:%s is already set to %s", repoName, tag, old) + } + store.Repositories[repoName] = repo + } + repo[tag] = img.Id + return store.Save() +} + +func (store *TagStore) Get(repoName string) (Repository, error) { + if err := store.Reload(); err != nil { + return nil, err + } + if r, exists := store.Repositories[repoName]; exists { + return r, nil + } + return nil, nil +} + +func (store *TagStore) GetImage(repoName, tag string) (*Image, error) { + repo, err := store.Get(repoName) + if err != nil { + return nil, err + } else if repo == nil { + return nil, nil + } + if revision, exists := repo[tag]; exists { + return store.graph.Get(revision) + } + return nil, nil +} + +// Validate the name of a repository +func validateRepoName(name string) error { + if name == "" { + return fmt.Errorf("Repository name can't be empty") + } + if strings.Contains(name, ":") { + return fmt.Errorf("Illegal repository name: %s", name) + } + return nil +} + +// Validate the name of a tag +func validateTagName(name string) error { + if name == "" { + return fmt.Errorf("Tag name can't be empty") + } + if strings.Contains(name, "/") || strings.Contains(name, ":") { + return fmt.Errorf("Illegal tag name: %s", name) + } + return nil +} diff --git a/utils.go b/utils.go index 520073e3ab..ce722f264f 100644 --- a/utils.go +++ b/utils.go @@ -3,13 +3,114 @@ package docker import ( "bytes" "container/list" + "errors" + "fmt" + "github.com/dotcloud/docker/rcli" "io" + "log" + "net/http" "os" "os/exec" "path/filepath" "sync" + "time" ) +// Go is a basic promise implementation: it wraps calls a function in a goroutine, +// and returns a channel which will later return the function's return value. +func Go(f func() error) chan error { + ch := make(chan error) + go func() { + ch <- f() + }() + return ch +} + +// Request a given URL and return an io.Reader +func Download(url string, stderr io.Writer) (*http.Response, error) { + var resp *http.Response + var err error = nil + if resp, err = http.Get(url); err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, errors.New("Got HTTP status code >= 400: " + resp.Status) + } + return resp, nil +} + +// Debug function, if the debug flag is set, then display. Do nothing otherwise +// If Docker is in damon mode, also send the debug info on the socket +func Debugf(format string, a ...interface{}) { + if rcli.DEBUG_FLAG { + log.Printf(format, a...) + if rcli.CLIENT_SOCKET != nil { + fmt.Fprintf(rcli.CLIENT_SOCKET, log.Prefix()+format, a...) + } + } +} + +// Reader with progress bar +type progressReader struct { + reader io.ReadCloser // Stream to read from + output io.Writer // Where to send progress bar to + read_total int // Expected stream length (bytes) + read_progress int // How much has been read so far (bytes) + last_update int // How many bytes read at least update +} + +func (r *progressReader) Read(p []byte) (n int, err error) { + read, err := io.ReadCloser(r.reader).Read(p) + r.read_progress += read + + // Only update progress for every 1% read + update_every := int(0.01 * float64(r.read_total)) + if r.read_progress-r.last_update > update_every || r.read_progress == r.read_total { + fmt.Fprintf(r.output, "%d/%d (%.0f%%)\r", + r.read_progress, + r.read_total, + float64(r.read_progress)/float64(r.read_total)*100) + r.last_update = r.read_progress + } + // Send newline when complete + if err == io.EOF { + fmt.Fprintf(r.output, "\n") + } + + return read, err +} +func (r *progressReader) Close() error { + return io.ReadCloser(r.reader).Close() +} +func ProgressReader(r io.ReadCloser, size int, output io.Writer) *progressReader { + return &progressReader{r, output, size, 0, 0} +} + +// HumanDuration returns a human-readable approximation of a duration +// (eg. "About a minute", "4 hours ago", etc.) +func HumanDuration(d time.Duration) string { + if seconds := int(d.Seconds()); seconds < 1 { + return "Less than a second" + } else if seconds < 60 { + return fmt.Sprintf("%d seconds", seconds) + } else if minutes := int(d.Minutes()); minutes == 1 { + return "About a minute" + } else if minutes < 60 { + return fmt.Sprintf("%d minutes", minutes) + } else if hours := int(d.Hours()); hours == 1 { + return "About an hour" + } else if hours < 48 { + return fmt.Sprintf("%d hours", hours) + } else if hours < 24*7*2 { + return fmt.Sprintf("%d days", hours/24) + } else if hours < 24*30*3 { + return fmt.Sprintf("%d weeks", hours/24/7) + } else if hours < 24*365*2 { + return fmt.Sprintf("%d months", hours/24/30) + } + return fmt.Sprintf("%d years", d.Hours()/24/365) +} + func Trunc(s string, maxlen int) string { if len(s) <= maxlen { return s