diff --git a/api.go b/api.go index 62969640e2..f520d7a73b 100644 --- a/api.go +++ b/api.go @@ -386,7 +386,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } sf := utils.NewStreamFormatter(version > 1.0) if image != "" { //pull - if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { + if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}, version > 1.3); err != nil { if sf.Used() { w.Write(sf.FormatError(err)) return nil diff --git a/buildfile.go b/buildfile.go index ba4427a49f..00c747d6d0 100644 --- a/buildfile.go +++ b/buildfile.go @@ -56,7 +56,7 @@ func (b *buildFile) CmdFrom(name string) error { if err != nil { if b.runtime.graph.IsNotExist(err) { remote, tag := utils.ParseRepositoryTag(name) - if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil); err != nil { + if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil, true); err != nil { return err } image, err = b.runtime.repositories.LookupImage(name) diff --git a/commands.go b/commands.go index ea38696e52..aa991811ca 100644 --- a/commands.go +++ b/commands.go @@ -30,8 +30,7 @@ import ( const VERSION = "0.5.0-dev" var ( - GITCOMMIT string - AuthRequiredError = fmt.Errorf("Authentication is required.") + GITCOMMIT string ) func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) { @@ -197,7 +196,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { // FIXME: ProgressReader shouldn't be this annoyning to use if context != nil { sf := utils.NewStreamFormatter(false) - body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s"), sf) + body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("", "Uploading context", "%v bytes%0.0s%0.0s"), sf) } // Upload the build context v := &url.Values{} @@ -846,7 +845,7 @@ func (cli *DockerCli) CmdPush(args ...string) error { } if err := push(); err != nil { - if err == AuthRequiredError { + if err == fmt.Errorf("Authentication is required.") { if err = cli.checkIfLogged("push"); err == nil { return push() } @@ -1569,19 +1568,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e } if resp.Header.Get("Content-Type") == "application/json" { - dec := json.NewDecoder(resp.Body) - for { - var jm utils.JSONMessage - if err := dec.Decode(&jm); err == io.EOF { - break - } else if err != nil { - return err - } - if jm.Error != nil && jm.Error.Code == 401 { - return AuthRequiredError - } - jm.Display(out) - } + return utils.DisplayJSONMessagesStream(resp.Body, out) } else { if _, err := io.Copy(out, resp.Body); err != nil { return err diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index 193be501d0..dd017cde8d 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -40,6 +40,10 @@ You can still call an old version of the api using What's new ---------- +.. http:post:: /images/create + + **New!** When pull a repo, all images are now downloaded in parallel. + .. http:get:: /containers/(id)/top **New!** You can now use ps args with docker top, like `docker top aux` diff --git a/graph.go b/graph.go index 1492735d7e..d02017bf8f 100644 --- a/graph.go +++ b/graph.go @@ -159,7 +159,7 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *uti if err != nil { return nil, err } - return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("Buffering to disk", "%v/%v (%v)"), sf), tmp.Root) + return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf), tmp.Root) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. diff --git a/runtime_test.go b/runtime_test.go index 512fc13782..8819df2221 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -101,7 +101,7 @@ func init() { // If the unit test is not found, try to download it. if img, err := globalRuntime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID { // Retrieve the Image - if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil { + if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil, true); err != nil { panic(err) } } diff --git a/server.go b/server.go index d77bcdb59e..7607851f49 100644 --- a/server.go +++ b/server.go @@ -145,7 +145,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. return "", err } - if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), path); err != nil { + if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("", "Downloading", "%8v/%v (%v)"), sf), path); err != nil { return "", err } // FIXME: Handle custom repo, tag comment, author @@ -153,7 +153,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. if err != nil { return "", err } - out.Write(sf.FormatStatus(img.ID)) + out.Write(sf.FormatStatus("", img.ID)) return img.ShortID(), nil } @@ -420,7 +420,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin // FIXME: Launch the getRemoteImage() in goroutines for _, id := range history { if !srv.runtime.graph.Exists(id) { - out.Write(sf.FormatStatus("Pulling %s metadata", id)) + out.Write(sf.FormatStatus(utils.TruncateID(id), "Pulling metadata")) imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token) if err != nil { // FIXME: Keep goging in case of error? @@ -432,13 +432,13 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin } // Get the layer - out.Write(sf.FormatStatus("Pulling %s fs layer", id)) + out.Write(sf.FormatStatus(utils.TruncateID(id), "Pulling fs layer")) layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token) if err != nil { return err } defer layer.Close() - if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), img); err != nil { + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf), img); err != nil { return err } } @@ -446,8 +446,8 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return nil } -func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error { - out.Write(sf.FormatStatus("Pulling repository %s", localName)) +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error { + out.Write(sf.FormatStatus("", "Pulling repository %s", localName)) repoData, err := r.GetRepositoryData(indexEp, remoteName) if err != nil { @@ -484,30 +484,51 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName repoData.ImgList[id].Tag = askedTag } - for _, img := range repoData.ImgList { - if askedTag != "" && img.Tag != askedTag { - utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) - continue + errors := make(chan error) + for _, image := range repoData.ImgList { + downloadImage := func(img *registry.ImgData) { + if askedTag != "" && img.Tag != askedTag { + utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) + errors <- nil + return + } + + if img.Tag == "" { + utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) + errors <- nil + return + } + out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Pulling image (%s) from %s", img.Tag, localName)) + success := false + for _, ep := range repoData.Endpoints { + if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { + out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) + continue + } + success = true + break + } + if !success { + errors <- fmt.Errorf("Could not find repository on any of the indexed registries.") + } + errors <- nil } - if img.Tag == "" { - utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) - continue - } - out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName)) - success := false - for _, ep := range repoData.Endpoints { - if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { - out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) - continue - } - success = true - break - } - if !success { - return fmt.Errorf("Could not find repository on any of the indexed registries.") + if parallel { + go downloadImage(image) + } else { + downloadImage(image) } } + + if parallel { + for i := 0; i < len(repoData.ImgList); i++ { + if err := <-errors; err != nil { + return err + } + } + } + for tag, id := range tagsList { if askedTag != "" && tag != askedTag { continue @@ -558,7 +579,7 @@ func (srv *Server) poolRemove(kind, key string) error { return nil } -func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { +func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig, parallel bool) error { r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory()) if err != nil { return err @@ -580,7 +601,7 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut } out = utils.NewWriteFlusher(out) - err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf) + err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf, parallel) if err != nil { if err := srv.pullImage(r, out, remoteName, endpoint, nil, sf); err != nil { return err @@ -620,12 +641,11 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) - imgList, err := srv.getImageList(localRepo) if err != nil { return err } - out.Write(sf.FormatStatus("Sending image list")) + out.Write(sf.FormatStatus("", "Sending image list")) var repoData *registry.RepositoryData repoData, err = r.PushImageJSONIndex(indexEp, remoteName, imgList, false, nil) @@ -634,14 +654,14 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName } for _, ep := range repoData.Endpoints { - out.Write(sf.FormatStatus("Pushing repository %s (%d tags)", localName, len(localRepo))) + out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", localName, len(localRepo))) // For each image within the repo, push them for _, elem := range imgList { if _, exists := repoData.ImgList[elem.ID]; exists { - out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) + out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID)) continue } else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) { - out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) + out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID)) continue } if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { @@ -650,7 +670,7 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName } else { elem.Checksum = checksum } - out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) + out.Write(sf.FormatStatus("", "Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { return err } @@ -670,7 +690,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, if err != nil { return "", fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) } - out.Write(sf.FormatStatus("Pushing %s", imgID)) + out.Write(sf.FormatStatus("", "Pushing %s", imgID)) imgData := ®istry.ImgData{ ID: imgID, @@ -679,7 +699,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, // Send the json if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { - out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID)) + out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", imgData.ID)) return "", nil } return "", err @@ -691,7 +711,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } // Send the layer - if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil { + if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil { return "", err } else { imgData.Checksum = checksum @@ -727,7 +747,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo if err != nil { reposLen := len(srv.runtime.repositories.Repositories[localName]) - out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", localName, reposLen)) + out.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", localName, reposLen)) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[localName]; exists { if err := srv.pushRepository(r, out, localName, remoteName, localRepo, endpoint, sf); err != nil { @@ -739,7 +759,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo } var token []string - out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName)) + out.Write(sf.FormatStatus("", "The push refers to an image: [%s]", localName)) if _, err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { return err } @@ -762,14 +782,14 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write u.Host = src u.Path = "" } - out.Write(sf.FormatStatus("Downloading from %s", u)) + out.Write(sf.FormatStatus("", "Downloading from %s", u)) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() resp, err = utils.Download(u.String(), out) if err != nil { return err } - archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("Importing", "%8v/%v (%v)"), sf) + archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("", "Importing", "%8v/%v (%v)"), sf) } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil) if err != nil { @@ -781,7 +801,7 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write return err } } - out.Write(sf.FormatStatus(img.ShortID())) + out.Write(sf.FormatStatus("", img.ShortID())) return nil } diff --git a/utils/utils.go b/utils/utils.go index 46914be0bd..e0083d02d8 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -94,11 +94,6 @@ func (r *progressReader) Read(p []byte) (n int, err error) { } r.lastUpdate = r.readProgress } - // Send newline when complete - if err != nil { - r.output.Write(r.sf.FormatStatus("")) - } - return read, err } func (r *progressReader) Close() error { @@ -107,7 +102,7 @@ func (r *progressReader) Close() error { func ProgressReader(r io.ReadCloser, size int, output io.Writer, template []byte, sf *StreamFormatter) *progressReader { tpl := string(template) if tpl == "" { - tpl = string(sf.FormatProgress("", "%8v/%v (%v)")) + tpl = string(sf.FormatProgress("", "%8v/%v (%v)", "")) } return &progressReader{r, NewWriteFlusher(output), size, 0, 0, tpl, sf} } @@ -587,11 +582,14 @@ type NopFlusher struct{} func (f *NopFlusher) Flush() {} type WriteFlusher struct { + sync.Mutex w io.Writer flusher http.Flusher } func (wf *WriteFlusher) Write(b []byte) (n int, err error) { + wf.Lock() + defer wf.Unlock() n, err = wf.w.Write(b) wf.flusher.Flush() return n, err @@ -633,17 +631,60 @@ func NewHTTPRequestError(msg string, res *http.Response) error { } func (jm *JSONMessage) Display(out io.Writer) error { + if jm.Error != nil { + if jm.Error.Code == 401 { + return fmt.Errorf("Authentication is required.") + } + return jm.Error + } + fmt.Fprintf(out, "%c[2K", 27) if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } + if jm.ID != "" { + fmt.Fprintf(out, "%s: ", jm.ID) + } if jm.Progress != "" { fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress) - } else if jm.Error != nil { - return jm.Error - } else if jm.ID != "" { - fmt.Fprintf(out, "%s: %s\n", jm.ID, jm.Status) } else { - fmt.Fprintf(out, "%s\n", jm.Status) + fmt.Fprintf(out, "%s\r", jm.Status) + } + if jm.ID == "" { + fmt.Fprintf(out, "\n") + } + return nil +} + +func DisplayJSONMessagesStream(in io.Reader, out io.Writer) error { + dec := json.NewDecoder(in) + jm := JSONMessage{} + ids := make(map[string]int) + diff := 0 + for { + if err := dec.Decode(&jm); err == io.EOF { + break + } else if err != nil { + return err + } + if jm.ID != "" { + line, ok := ids[jm.ID] + if !ok { + line = len(ids) + ids[jm.ID] = line + fmt.Fprintf(out, "\n") + diff = 0 + } else { + diff = len(ids) - line + } + fmt.Fprintf(out, "%c[%dA", 27, diff) + } + err := jm.Display(out) + if jm.ID != "" { + fmt.Fprintf(out, "%c[%dB", 27, diff) + } + if err != nil { + return err + } } return nil } @@ -657,11 +698,11 @@ func NewStreamFormatter(json bool) *StreamFormatter { return &StreamFormatter{json, false} } -func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte { +func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []byte { sf.used = true str := fmt.Sprintf(format, a...) if sf.json { - b, err := json.Marshal(&JSONMessage{Status: str}) + b, err := json.Marshal(&JSONMessage{ID: id, Status: str}) if err != nil { return sf.FormatError(err) } @@ -685,16 +726,16 @@ func (sf *StreamFormatter) FormatError(err error) []byte { return []byte("Error: " + err.Error() + "\r\n") } -func (sf *StreamFormatter) FormatProgress(action, str string) []byte { +func (sf *StreamFormatter) FormatProgress(id, action, progress string) []byte { sf.used = true if sf.json { - b, err := json.Marshal(&JSONMessage{Status: action, Progress: str}) + b, err := json.Marshal(&JSONMessage{Status: action, Progress: progress, ID: id}) if err != nil { return nil } return b } - return []byte(action + " " + str + "\r") + return []byte(action + " " + progress + "\r") } func (sf *StreamFormatter) Used() bool {