From 2421838b0ac6a496503bc896da0e60d195132576 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 15 Apr 2013 03:28:54 -0700 Subject: [PATCH 01/36] Support for the new registry/index API (wip) --- auth/auth.go | 6 ++--- registry.go | 64 +++++++++++++++++++++++++++------------------------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 2eb63b1d4a..904198da5f 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -16,7 +16,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const REGISTRY_SERVER = "https://registry.docker.io" +const INDEX_SERVER = "https://index.docker.io" type AuthConfig struct { Username string `json:"username"` @@ -113,7 +113,7 @@ func Login(authConfig *AuthConfig) (string, error) { // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := http.Post(REGISTRY_SERVER+"/v1/users", "application/json; charset=utf-8", b) + req1, err := http.Post(INDEX_SERVER+"/v1/users", "application/json; charset=utf-8", b) if err != nil { errMsg = fmt.Sprintf("Server Error: %s", err) return "", errors.New(errMsg) @@ -134,7 +134,7 @@ func Login(authConfig *AuthConfig) (string, error) { // FIXME: This should be 'exists', not 'exist'. Need to change on the server first. if string(reqBody) == "Username or email already exist" { client := &http.Client{} - req, err := http.NewRequest("GET", REGISTRY_SERVER+"/v1/users", nil) + req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { diff --git a/registry.go b/registry.go index 74b166906f..73bf6a1f03 100644 --- a/registry.go +++ b/registry.go @@ -13,8 +13,8 @@ import ( ) //FIXME: Set the endpoint in a conf file or via commandline -//const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1" -const REGISTRY_ENDPOINT = auth.REGISTRY_SERVER + "/v1" +//const INDEX_ENDPOINT = "http://registry-creack.dotcloud.com/v1" +const INDEX_ENDPOINT = auth.INDEX_SERVER + "/v1" // Build an Image object from raw json data func NewImgJson(src []byte) (*Image, error) { @@ -48,10 +48,10 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) ([]*Image, error) { +func (graph *Graph) getRemoteHistory(imgId, registry string, authConfig *auth.AuthConfig) ([]*Image, error) { client := &http.Client{} - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/history", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } @@ -78,29 +78,26 @@ func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) } // Check if an image exists in the Registry -func (graph *Graph) LookupRemoteImage(imgId string, authConfig *auth.AuthConfig) bool { +func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return false } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := rt.RoundTrip(req) - if err != nil || res.StatusCode != 307 { - return false - } - return res.StatusCode == 307 + return err == nil && res.StatusCode == 307 } // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) -func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *auth.AuthConfig) (*Image, Archive, error) { +func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, authConfig *auth.AuthConfig) (*Image, Archive, error) { client := &http.Client{} fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) // Get the Json - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) } @@ -127,7 +124,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *a // Get the layer fmt.Fprintf(stdout, "Pulling %s fs layer\r\n", imgId) - req, err = http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/layer", nil) + req, err = http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } @@ -139,7 +136,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *a return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil } -func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.AuthConfig) error { +func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, authConfig *auth.AuthConfig) error { history, err := graph.getRemoteHistory(imgId, authConfig) if err != nil { return err @@ -148,7 +145,7 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.A // FIXME: Lunch the getRemoteImage() in goroutines for _, j := range history { if !graph.Exists(j.Id) { - img, layer, err := graph.getRemoteImage(stdout, j.Id, authConfig) + img, layer, err := graph.getRemoteImage(stdout, j.Id, registry, authConfig) if err != nil { // FIXME: Keep goging in case of error? return err @@ -162,7 +159,7 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.A } // FIXME: Handle the askedTag parameter -func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { +func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag, registry string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) @@ -170,9 +167,9 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re var repositoryTarget string // If we are asking for 'root' repository, lookup on the Library's registry if strings.Index(remote, "/") == -1 { - repositoryTarget = REGISTRY_ENDPOINT + "/library/" + remote + repositoryTarget = registry + "/library/" + remote } else { - repositoryTarget = REGISTRY_ENDPOINT + "/users/" + remote + repositoryTarget = registry + "/users/" + remote } req, err := http.NewRequest("GET", repositoryTarget, nil) @@ -198,7 +195,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re } for tag, rev := range t { fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) - if err = graph.PullImage(stdout, rev, authConfig); err != nil { + if err = graph.PullImage(stdout, rev, registry, authConfig); err != nil { return err } if err = repositories.Set(remote, tag, rev, true); err != nil { @@ -303,7 +300,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth // push a tag on the registry. // Remote has the format '/ -func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig *auth.AuthConfig) error { // Keep this for backward compatibility if tag == "" { @@ -313,10 +310,10 @@ func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthC // "jsonify" the string revision = "\"" + revision + "\"" - Debugf("Pushing tags for rev [%s] on {%s}\n", revision, REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag) + Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) client := &http.Client{} - req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag, strings.NewReader(revision)) + req, err := http.NewRequest("PUT", registry+"/users/"+remote+"/"+tag, strings.NewReader(revision)) req.Header.Add("Content-type", "application/json") req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) @@ -337,15 +334,20 @@ func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthC return nil } -func (graph *Graph) LookupRemoteRepository(remote string, authConfig *auth.AuthConfig) bool { +func (graph *Graph) LookupRemoteRepository(remote, registry string, authConfig *auth.AuthConfig) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} var repositoryTarget string // If we are asking for 'root' repository, lookup on the Library's registry if strings.Index(remote, "/") == -1 { - repositoryTarget = REGISTRY_ENDPOINT + "/library/" + remote + "/lookup" + repositoryTarget = registry + "/library/" + remote + "/lookup" } else { - repositoryTarget = REGISTRY_ENDPOINT + "/users/" + remote + "/lookup" + parts := strings.Split(remote, "/") + if len(parts) != 2 { + Debugf("Repository must abide to following format: user/repo_name") + return false + } + repositoryTarget = registry + "/users/" + parts[0] + "/repositories/" + parts[1] } Debugf("Checking for permissions on: %s", repositoryTarget) req, err := http.NewRequest("PUT", repositoryTarget, strings.NewReader("\"\"")) @@ -368,7 +370,7 @@ func (graph *Graph) LookupRemoteRepository(remote string, authConfig *auth.AuthC } // FIXME: this should really be PushTag -func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, authConfig *auth.AuthConfig) error { // Check if the local impage exists img, err := graph.Get(imgId) if err != nil { @@ -377,12 +379,12 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, a } fmt.Fprintf(stdout, "Pushing tag %s:%s\r\n", remote, tag) // Push the image - if err = graph.PushImage(stdout, img, authConfig); err != nil { + if err = graph.PushImage(stdout, img, registry, authConfig); err != nil { return err } fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) // And then the tag - if err = graph.pushTag(remote, imgId, tag, authConfig); err != nil { + if err = graph.pushTag(remote, imgId, registry, tag, authConfig); err != nil { return err } return nil @@ -390,16 +392,16 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, a // Push a repository to the registry. // Remote has the format '/ -func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { +func (graph *Graph) PushRepository(stdout io.Writer, remote, registry string, localRepo Repository, authConfig *auth.AuthConfig) error { // Check if the remote repository exists/if we have the permission - if !graph.LookupRemoteRepository(remote, authConfig) { + if !graph.LookupRemoteRepository(remote, registry, authConfig) { return fmt.Errorf("Permission denied on repository %s\n", remote) } fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) // For each image within the repo, push them for tag, imgId := range localRepo { - if err := graph.pushPrimitive(stdout, remote, tag, imgId, authConfig); err != nil { + if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, authConfig); err != nil { // FIXME: Continue on error? return err } From 2f082510a70a19fa83a7214485e3838e15bde22f Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 15 Apr 2013 11:17:03 -0700 Subject: [PATCH 02/36] Implemented new version of PullRepository. Missing support for whole repository pull (= no tag specified) --- commands.go | 8 ++- registry.go | 171 ++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 144 insertions(+), 35 deletions(-) diff --git a/commands.go b/commands.go index 4be282bce2..dc12935265 100644 --- a/commands.go +++ b/commands.go @@ -553,6 +553,8 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ... func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "NAME", "Pull an image or a repository from the registry") + tag := cmd.String("t", "", "Download tagged image in repository") + registry := cmd.String("registry", "", "Registry to download from. Necessary if image is pulled by ID") if err := cmd.Parse(args); err != nil { return nil } @@ -563,14 +565,14 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string } // FIXME: CmdPull should be a wrapper around Runtime.Pull() - if srv.runtime.graph.LookupRemoteImage(remote, srv.runtime.authConfig) { - if err := srv.runtime.graph.PullImage(stdout, remote, srv.runtime.authConfig); err != nil { + if *registry != "" { + if err := srv.runtime.graph.PullImage(stdout, remote, *registry, nil); err != nil { return err } return nil } // FIXME: Allow pull repo:tag - if err := srv.runtime.graph.PullRepository(stdout, remote, "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { + if err := srv.runtime.graph.PullRepository(stdout, remote, *tag, srv.runtime.repositories, srv.runtime.authConfig); err != nil { return err } return nil diff --git a/registry.go b/registry.go index 73bf6a1f03..07e79a5a5f 100644 --- a/registry.go +++ b/registry.go @@ -48,14 +48,15 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (graph *Graph) getRemoteHistory(imgId, registry string, authConfig *auth.AuthConfig) ([]*Image, error) { +func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) { client := &http.Client{} req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token + // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -92,7 +93,7 @@ func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.A // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) -func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, authConfig *auth.AuthConfig) (*Image, Archive, error) { +func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, token []string) (*Image, Archive, error) { client := &http.Client{} fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) @@ -101,7 +102,8 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, aut if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token + // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -128,7 +130,8 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, aut if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token + // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -136,16 +139,16 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, aut return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil } -func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, authConfig *auth.AuthConfig) error { - history, err := graph.getRemoteHistory(imgId, authConfig) +func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token []string) error { + history, err := graph.getRemoteHistory(imgId, registry, token) if err != nil { return err } // FIXME: Try to stream the images? - // FIXME: Lunch the getRemoteImage() in goroutines + // FIXME: Launch the getRemoteImage() in goroutines for _, j := range history { if !graph.Exists(j.Id) { - img, layer, err := graph.getRemoteImage(stdout, j.Id, registry, authConfig) + img, layer, err := graph.getRemoteImage(stdout, j.Id, registry, token) if err != nil { // FIXME: Keep goging in case of error? return err @@ -158,58 +161,162 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, authConf return nil } -// FIXME: Handle the askedTag parameter -func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag, registry string, repositories *TagStore, authConfig *auth.AuthConfig) error { +// // FIXME: Handle the askedTag parameter +// func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag, registry string, repositories *TagStore, authConfig *auth.AuthConfig) error { +// client := &http.Client{} + +// fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) + +// var repositoryTarget string +// // If we are asking for 'root' repository, lookup on the Library's registry +// if strings.Index(remote, "/") == -1 { +// repositoryTarget = registry + "/library/" + remote +// } else { +// repositoryTarget = registry + "/users/" + remote +// } + +// req, err := http.NewRequest("GET", repositoryTarget, nil) +// if err != nil { +// return err +// } +// req.SetBasicAuth(authConfig.Username, authConfig.Password) +// res, err := client.Do(req) +// if err != nil { +// return err +// } +// defer res.Body.Close() +// if res.StatusCode != 200 { +// return fmt.Errorf("HTTP code: %d", res.StatusCode) +// } +// rawJson, err := ioutil.ReadAll(res.Body) +// if err != nil { +// return err +// } +// t := map[string]string{} +// if err = json.Unmarshal(rawJson, &t); err != nil { +// return err +// } +// for tag, rev := range t { +// fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) +// if err = graph.PullImage(stdout, rev, registry, authConfig); err != nil { +// return err +// } +// if err = repositories.Set(remote, tag, rev, true); err != nil { +// return err +// } +// } +// if err = repositories.Save(); err != nil { +// return err +// } +// return nil +// } + +func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - var repositoryTarget string // If we are asking for 'root' repository, lookup on the Library's registry if strings.Index(remote, "/") == -1 { - repositoryTarget = registry + "/library/" + remote + repositoryTarget = INDEX_ENDPOINT + "/repositories/library/" + remote + "/checksums" } else { - repositoryTarget = registry + "/users/" + remote + repositoryTarget = INDEX_ENDPOINT + "/repositories/" + remote + "/checksums" } req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { return err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + if authConfig != nil { + req.SetBasicAuth(authConfig.Username, authConfig.Password) + } + res, err := client.Do(req) if err != nil { return err } defer res.Body.Close() + // TODO: Right now we're ignoring checksums in the response body. + // In the future, we need to use them to check image validity. if res.StatusCode != 200 { return fmt.Errorf("HTTP code: %d", res.StatusCode) } - rawJson, err := ioutil.ReadAll(res.Body) - if err != nil { - return err + + var token, endpoints []string + if res.Header.Get("X-Docker-Token") != "" { + token = res.Header["X-Docker-Token"] } - t := map[string]string{} - if err = json.Unmarshal(rawJson, &t); err != nil { - return err + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return fmt.Errorf("Index response didn't contain any endpoints") } - for tag, rev := range t { - fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) - if err = graph.PullImage(stdout, rev, registry, authConfig); err != nil { + + // FIXME: If askedTag is empty, fetch all tags. + if askedTag == "" { + askedTag = "latest" + } + + for _, registry := range endpoints { + registryEndpoint := "https://" + registry + "/v1" + if strings.Index(remote, "/") == -1 { + repositoryTarget = registryEndpoint + "/repositories/library/" + + remote + "/tags/" + askedTag + } else { + repositoryTarget = registryEndpoint + "/repositories/users/" + + remote + "/tags/" + askedTag + } + + req, err = http.NewRequest("GET", repositoryTarget, nil) + if err != nil { return err } - if err = repositories.Set(remote, tag, rev, true); err != nil { + req.Header["X-Docker-Token"] = token + res, err := client.Do(req) + if err != nil { + fmt.Fprintf(stdout, "Error while retrieving repository info: %v ; " + + "checking next endpoint") + continue + } + defer res.Body.Close() + if res.StatusCode == 403 { + if authConfig == nil { + return fmt.Errorf("You need to be authenticated to access this resource") + } else { + return fmt.Errorf("You aren't authorized to access this resource") + } + } else if res.StatusCode != 200 { + return fmt.Errorf("HTTP code: %d", res.StatusCode) + } + + var imgId string + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { return err } + if err = json.Unmarshal(rawJson, &imgId); err != nil { + return err + } + + if err := graph.PullImage(stdout, imgId, registryEndpoint, token); err != nil { + return err + } + + if err = repositories.Set(remote, askedTag, imgId, true); err != nil { + return err + } + + if err = repositories.Save(); err != nil { + return err + } + + return nil } - if err = repositories.Save(); err != nil { - return err - } - return nil + return fmt.Errorf("Could not find repository on any of the indexed registries.") } // Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth.AuthConfig) error { +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, authConfig *auth.AuthConfig) error { client := &http.Client{} // FIXME: Factorize the code @@ -225,7 +332,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth // FIXME: try json with UTF8 jsonData := strings.NewReader(string(jsonRaw)) - req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData) + req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) if err != nil { return err } @@ -252,7 +359,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth } fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil) + req2, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", nil) req2.SetBasicAuth(authConfig.Username, authConfig.Password) res2, err := client.Do(req2) if err != nil { From e639309a7a9c12fe713857a988806285889b580c Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 17 Apr 2013 10:21:32 -0700 Subject: [PATCH 03/36] Reimplemented feature: downloading all tags on a repository using docker pull. Temporarily commented out CmdPush --- commands.go | 102 +++++++++++++------------- registry.go | 207 +++++++++++++++++++++++++++------------------------- 2 files changed, 157 insertions(+), 152 deletions(-) diff --git a/commands.go b/commands.go index dc12935265..8671243f9f 100644 --- a/commands.go +++ b/commands.go @@ -492,64 +492,64 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout rcli.DockerConn, args . return nil } -func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { - cmd := rcli.Subcmd(stdout, "push", "NAME", "Push an image or a repository to the registry") - if err := cmd.Parse(args); err != nil { - return nil - } - local := cmd.Arg(0) +// func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { +// cmd := rcli.Subcmd(stdout, "push", "NAME", "Push an image or a repository to the registry") +// if err := cmd.Parse(args); err != nil { +// return nil +// } +// local := cmd.Arg(0) - if local == "" { - cmd.Usage() - return nil - } +// if local == "" { +// cmd.Usage() +// return nil +// } - // If the login failed, abort - if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { - if err := srv.CmdLogin(stdin, stdout, args...); err != nil { - return err - } - if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { - return fmt.Errorf("Please login prior to push. ('docker login')") - } - } +// // If the login failed, abort +// if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { +// if err := srv.CmdLogin(stdin, stdout, args...); err != nil { +// return err +// } +// if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { +// return fmt.Errorf("Please login prior to push. ('docker login')") +// } +// } - var remote string +// var remote string - tmp := strings.SplitN(local, "/", 2) - if len(tmp) == 1 { - return fmt.Errorf( - "Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", - srv.runtime.authConfig.Username, local) - } else { - remote = local - } +// tmp := strings.SplitN(local, "/", 2) +// if len(tmp) == 1 { +// return fmt.Errorf( +// "Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", +// srv.runtime.authConfig.Username, local) +// } else { +// remote = local +// } - Debugf("Pushing [%s] to [%s]\n", local, remote) +// Debugf("Pushing [%s] to [%s]\n", local, remote) - // Try to get the image - // FIXME: Handle lookup - // FIXME: Also push the tags in case of ./docker push myrepo:mytag - // img, err := srv.runtime.LookupImage(cmd.Arg(0)) - img, err := srv.runtime.graph.Get(local) - if err != nil { - Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) - // If it fails, try to get the repository - if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { - if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { - return err - } - return nil - } +// // Try to get the image +// // FIXME: Handle lookup +// // FIXME: Also push the tags in case of ./docker push myrepo:mytag +// // img, err := srv.runtime.LookupImage(cmd.Arg(0)) +// img, err := srv.runtime.graph.Get(local) +// if err != nil { +// Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) +// // If it fails, try to get the repository +// if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { +// if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { +// return err +// } +// return nil +// } - return err - } - err = srv.runtime.graph.PushImage(stdout, img, srv.runtime.authConfig) - if err != nil { - return err - } - return nil -} +// return err +// } +// err = srv.runtime.graph.PushImage(stdout, img, srv.runtime.authConfig) +// if err != nil { +// return err +// } +// return nil +// } func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "NAME", "Pull an image or a repository from the registry") diff --git a/registry.go b/registry.go index 07e79a5a5f..edaaeed502 100644 --- a/registry.go +++ b/registry.go @@ -139,6 +139,79 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil } +func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { + client := &http.Client{} + for _, host := range registries { + endpoint := "https://" + host + "/v1/repositories/users/" + repository + "/tags" + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, err + } + req.Header["X-Docker-Token"] = token + res, err := client.Do(req) + defer res.Body.Close() + if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { + Debugf("Registry isn't responding: trying another registry endpoint") + continue + } else if res.StatusCode == 404 { + return nil, fmt.Errorf("Repository not found") + } + + var result *map[string]string + + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + if err = json.Unmarshal(rawJson, result); err != nil { + return nil, err + } + + return *result, nil + + } + return nil, fmt.Errorf("Could not reach any registry endpoint") +} + +func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry string, token []string) (string, error) { + client := &http.Client{} + registryEndpoint := "https://" + registry + "/v1" + var repositoryTarget string + if strings.Index(remote, "/") == -1 { + repositoryTarget = registryEndpoint + "/repositories/library/" + + remote + "/tags/" + tag + } else { + repositoryTarget = registryEndpoint + "/repositories/users/" + + remote + "/tags/" + tag + } + + req, err := http.NewRequest("GET", repositoryTarget, nil) + if err != nil { + return "", err + } + req.Header["X-Docker-Token"] = token + res, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("Error while retrieving repository info: %v", err) + } + defer res.Body.Close() + if res.StatusCode == 403 { + return "", fmt.Errorf("You aren't authorized to access this resource") + } else if res.StatusCode != 200 { + return "", fmt.Errorf("HTTP code: %d", res.StatusCode) + } + + var imgId string + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + if err = json.Unmarshal(rawJson, &imgId); err != nil { + return "", err + } + return imgId, nil +} + func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token []string) error { history, err := graph.getRemoteHistory(imgId, registry, token) if err != nil { @@ -161,56 +234,6 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token [] return nil } -// // FIXME: Handle the askedTag parameter -// func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag, registry string, repositories *TagStore, authConfig *auth.AuthConfig) error { -// client := &http.Client{} - -// fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - -// var repositoryTarget string -// // If we are asking for 'root' repository, lookup on the Library's registry -// if strings.Index(remote, "/") == -1 { -// repositoryTarget = registry + "/library/" + remote -// } else { -// repositoryTarget = registry + "/users/" + remote -// } - -// req, err := http.NewRequest("GET", repositoryTarget, nil) -// if err != nil { -// return err -// } -// req.SetBasicAuth(authConfig.Username, authConfig.Password) -// res, err := client.Do(req) -// if err != nil { -// return err -// } -// defer res.Body.Close() -// if res.StatusCode != 200 { -// return fmt.Errorf("HTTP code: %d", res.StatusCode) -// } -// rawJson, err := ioutil.ReadAll(res.Body) -// if err != nil { -// return err -// } -// t := map[string]string{} -// if err = json.Unmarshal(rawJson, &t); err != nil { -// return err -// } -// for tag, rev := range t { -// fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) -// if err = graph.PullImage(stdout, rev, registry, authConfig); err != nil { -// return err -// } -// if err = repositories.Set(remote, tag, rev, true); err != nil { -// return err -// } -// } -// if err = repositories.Save(); err != nil { -// return err -// } -// return nil -// } - func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} @@ -253,66 +276,48 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re } // FIXME: If askedTag is empty, fetch all tags. + var tagsList map[string]string if askedTag == "" { - askedTag = "latest" - } - - for _, registry := range endpoints { - registryEndpoint := "https://" + registry + "/v1" - if strings.Index(remote, "/") == -1 { - repositoryTarget = registryEndpoint + "/repositories/library/" + - remote + "/tags/" + askedTag - } else { - repositoryTarget = registryEndpoint + "/repositories/users/" + - remote + "/tags/" + askedTag - } - - req, err = http.NewRequest("GET", repositoryTarget, nil) + tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) if err != nil { return err } - req.Header["X-Docker-Token"] = token - res, err := client.Do(req) - if err != nil { - fmt.Fprintf(stdout, "Error while retrieving repository info: %v ; " + - "checking next endpoint") - continue - } - defer res.Body.Close() - if res.StatusCode == 403 { - if authConfig == nil { - return fmt.Errorf("You need to be authenticated to access this resource") - } else { - return fmt.Errorf("You aren't authorized to access this resource") + } else { + tagsList = map[string]string{ askedTag : "" } + } + + for askedTag, imgId := range tagsList { + success := false + for _, registry := range endpoints { + if imgId == "" { + imgId, err = graph.getImageForTag(stdout, askedTag, remote, registry, token) + if err != nil { + fmt.Fprintf(stdout, "Error while retrieving image for tag: %v (%v) ; " + + "checking next endpoint", askedTag, err) + continue + } } - } else if res.StatusCode != 200 { - return fmt.Errorf("HTTP code: %d", res.StatusCode) + + if err := graph.PullImage(stdout, imgId, "https://" + registry + "/v1", token); err != nil { + return err + } + + if err = repositories.Set(remote, askedTag, imgId, true); err != nil { + return err + } + success = true } - var imgId string - rawJson, err := ioutil.ReadAll(res.Body) - if err != nil { - return err + if !success { + return fmt.Errorf("Could not find repository on any of the indexed registries.") } - if err = json.Unmarshal(rawJson, &imgId); err != nil { - return err - } - - if err := graph.PullImage(stdout, imgId, registryEndpoint, token); err != nil { - return err - } - - if err = repositories.Set(remote, askedTag, imgId, true); err != nil { - return err - } - - if err = repositories.Save(); err != nil { - return err - } - - return nil } - return fmt.Errorf("Could not find repository on any of the indexed registries.") + + if err = repositories.Save(); err != nil { + return err + } + + return nil } // Push a local image to the registry with its history if needed From 1cf8a2c26c8ae663dea9a613dfb236e6c2257550 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 18 Apr 2013 08:16:58 -0700 Subject: [PATCH 04/36] Changed some of the routes to reflect changes made to the API ; added HTTPClient singleton to the graph object --- graph.go | 3 +- registry.go | 97 ++++++++++++++++------------------------------------- 2 files changed, 30 insertions(+), 70 deletions(-) diff --git a/graph.go b/graph.go index 3823868c83..1b2c441974 100644 --- a/graph.go +++ b/graph.go @@ -2,7 +2,7 @@ package docker import ( "fmt" - "io" + "net/http" "io/ioutil" "os" "path" @@ -15,6 +15,7 @@ import ( type Graph struct { Root string idIndex *TruncIndex + httpClient *http.Client } // NewGraph instantiates a new graph at the given root path in the filesystem. diff --git a/registry.go b/registry.go index edaaeed502..aacdf8357a 100644 --- a/registry.go +++ b/registry.go @@ -49,7 +49,7 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) { - client := &http.Client{} + client := graph.getHttpClient() req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { @@ -78,6 +78,13 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] return history, nil } +func (graph *Graph) getHttpClient() *http.Client { + if graph.httpClient == nil { + graph.httpClient = new(http.Client) + } + return graph.httpClient +} + // Check if an image exists in the Registry func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} @@ -94,7 +101,7 @@ func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.A // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, token []string) (*Image, Archive, error) { - client := &http.Client{} + client := graph.getHttpClient() fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) // Get the Json @@ -140,9 +147,9 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok } func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { - client := &http.Client{} + client := graph.getHttpClient() for _, host := range registries { - endpoint := "https://" + host + "/v1/repositories/users/" + repository + "/tags" + endpoint := "https://" + host + "/v1/repositories/" + repository + "/tags" req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -174,16 +181,9 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit } func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry string, token []string) (string, error) { - client := &http.Client{} + client := graph.getHttpClient() registryEndpoint := "https://" + registry + "/v1" - var repositoryTarget string - if strings.Index(remote, "/") == -1 { - repositoryTarget = registryEndpoint + "/repositories/library/" + - remote + "/tags/" + tag - } else { - repositoryTarget = registryEndpoint + "/repositories/users/" + - remote + "/tags/" + tag - } + repositoryTarget := registryEndpoint + "/repositories/" + remote + "/tags/" + tag req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -235,16 +235,10 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token [] } func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { - client := &http.Client{} + client := graph.getHttpClient() fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - var repositoryTarget string - // If we are asking for 'root' repository, lookup on the Library's registry - if strings.Index(remote, "/") == -1 { - repositoryTarget = INDEX_ENDPOINT + "/repositories/library/" + remote + "/checksums" - } else { - repositoryTarget = INDEX_ENDPOINT + "/repositories/" + remote + "/checksums" - } + repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/checksums" req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -322,7 +316,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re // Push a local image to the registry with its history if needed func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, authConfig *auth.AuthConfig) error { - client := &http.Client{} + client := graph.getHttpClient() // FIXME: Factorize the code // FIXME: Do the puts in goroutines @@ -424,7 +418,7 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig * Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) - client := &http.Client{} + client := graph.getHttpClient() req, err := http.NewRequest("PUT", registry+"/users/"+remote+"/"+tag, strings.NewReader(revision)) req.Header.Add("Content-type", "application/json") req.SetBasicAuth(authConfig.Username, authConfig.Password) @@ -446,41 +440,6 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig * return nil } -func (graph *Graph) LookupRemoteRepository(remote, registry string, authConfig *auth.AuthConfig) bool { - rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - - var repositoryTarget string - // If we are asking for 'root' repository, lookup on the Library's registry - if strings.Index(remote, "/") == -1 { - repositoryTarget = registry + "/library/" + remote + "/lookup" - } else { - parts := strings.Split(remote, "/") - if len(parts) != 2 { - Debugf("Repository must abide to following format: user/repo_name") - return false - } - repositoryTarget = registry + "/users/" + parts[0] + "/repositories/" + parts[1] - } - Debugf("Checking for permissions on: %s", repositoryTarget) - req, err := http.NewRequest("PUT", repositoryTarget, strings.NewReader("\"\"")) - if err != nil { - Debugf("%s\n", err) - return false - } - req.SetBasicAuth(authConfig.Username, authConfig.Password) - req.Header.Add("Content-type", "application/json") - res, err := rt.RoundTrip(req) - if err != nil || res.StatusCode != 404 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - errBody = []byte(err.Error()) - } - Debugf("Lookup status code: %d (body: %s)", res.StatusCode, errBody) - return false - } - return true -} - // FIXME: this should really be PushTag func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, authConfig *auth.AuthConfig) error { // Check if the local impage exists @@ -506,17 +465,17 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry // Remote has the format '/ func (graph *Graph) PushRepository(stdout io.Writer, remote, registry string, localRepo Repository, authConfig *auth.AuthConfig) error { // Check if the remote repository exists/if we have the permission - if !graph.LookupRemoteRepository(remote, registry, authConfig) { - return fmt.Errorf("Permission denied on repository %s\n", remote) - } + // if !graph.LookupRemoteRepository(remote, registry, authConfig) { + // return fmt.Errorf("Permission denied on repository %s\n", remote) + // } - fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) - // For each image within the repo, push them - for tag, imgId := range localRepo { - if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, authConfig); err != nil { - // FIXME: Continue on error? - return err - } - } + // fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) + // // For each image within the repo, push them + // for tag, imgId := range localRepo { + // if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, authConfig); err != nil { + // // FIXME: Continue on error? + // return err + // } + // } return nil } From 7c1a27e2add585f22a833020dd0551a892b3af69 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 18 Apr 2013 08:34:43 -0700 Subject: [PATCH 05/36] gofmt pass --- graph.go | 6 +++--- registry.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/graph.go b/graph.go index 1b2c441974..ea0c42b3d2 100644 --- a/graph.go +++ b/graph.go @@ -2,8 +2,8 @@ package docker import ( "fmt" - "net/http" "io/ioutil" + "net/http" "os" "path" "path/filepath" @@ -13,8 +13,8 @@ import ( // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *TruncIndex + Root string + idIndex *TruncIndex httpClient *http.Client } diff --git a/registry.go b/registry.go index aacdf8357a..6ae4a425d7 100644 --- a/registry.go +++ b/registry.go @@ -277,7 +277,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return err } } else { - tagsList = map[string]string{ askedTag : "" } + tagsList = map[string]string{askedTag: ""} } for askedTag, imgId := range tagsList { @@ -286,13 +286,13 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re if imgId == "" { imgId, err = graph.getImageForTag(stdout, askedTag, remote, registry, token) if err != nil { - fmt.Fprintf(stdout, "Error while retrieving image for tag: %v (%v) ; " + - "checking next endpoint", askedTag, err) + fmt.Fprintf(stdout, "Error while retrieving image for tag: %v (%v) ; "+ + "checking next endpoint", askedTag, err) continue } } - if err := graph.PullImage(stdout, imgId, "https://" + registry + "/v1", token); err != nil { + if err := graph.PullImage(stdout, imgId, "https://"+registry+"/v1", token); err != nil { return err } From 048fd671ef107aa14f02e6fe2db4a4100c23d37f Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 19 Apr 2013 08:25:55 -0700 Subject: [PATCH 06/36] Implemented checksum computation on image creation (necessary for new push primitive) --- container.go | 13 +++++++++++++ graph.go | 10 ++++++---- image.go | 1 + 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/container.go b/container.go index bac0951da4..891ae750ae 100644 --- a/container.go +++ b/container.go @@ -1,6 +1,7 @@ package docker import ( + "crypto/sha256" "encoding/json" "fmt" "github.com/dotcloud/docker/rcli" @@ -695,6 +696,18 @@ func (container *Container) ExportRw() (Archive, error) { return Tar(container.rwPath(), Uncompressed) } +func (container *Container) RwChecksum() (string, error) { + h := sha256.New() + rwData, err := container.ExportRw() + if err != nil { + return "", err + } + if _, err := io.Copy(h, rwData); err != nil { + return "", err + } + return string(h.Sum(nil)), nil +} + func (container *Container) Export() (Archive, error) { if err := container.EnsureMounted(); err != nil { return nil, err diff --git a/graph.go b/graph.go index ea0c42b3d2..59a8cba8db 100644 --- a/graph.go +++ b/graph.go @@ -98,11 +98,13 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Parent = container.Image img.Container = container.Id img.ContainerConfig = *container.Config - if config == nil { - if parentImage, err := graph.Get(container.Image); err == nil && parentImage != nil { - img.Config = parentImage.Config - } + // FIXME: If an image is pulled from a raw URL (not created from a container), + // its checksum will not be computed, which will cause a push to fail + checksum, err := container.RwChecksum() + if err != nil { + return nil, err } + img.Checksum = checksum } if err := graph.Register(layerData, img); err != nil { return nil, err diff --git a/image.go b/image.go index 09c0f8dcf6..8090e583e6 100644 --- a/image.go +++ b/image.go @@ -18,6 +18,7 @@ import ( type Image struct { Id string `json:"id"` Parent string `json:"parent,omitempty"` + Checksum string `json:"checksum,omitempty"` Comment string `json:"comment,omitempty"` Created time.Time `json:"created"` Container string `json:"container,omitempty"` From e179c66400eeafd732218e981c5aaccd5a0b72c4 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 23 Apr 2013 09:52:46 -0700 Subject: [PATCH 07/36] Reimplemented docker pull for new registry API (command is still deactivated) --- graph.go | 27 ++++++++++ registry.go | 139 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 114 insertions(+), 52 deletions(-) diff --git a/graph.go b/graph.go index 59a8cba8db..361a70912f 100644 --- a/graph.go +++ b/graph.go @@ -292,3 +292,30 @@ func (graph *Graph) Heads() (map[string]*Image, error) { func (graph *Graph) imageRoot(id string) string { return path.Join(graph.Root, id) } + +func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { + var checksums map[string]string + var result []map[string]string + for _, id := range repo { + img, err := graph.Get(id) + if err != nil { + return nil, err + } + err = img.WalkHistory(func(image *Image) error { + checksums[image.Id] = image.Checksum + return nil + }) + if err != nil { + return nil, err + } + } + i := 0 + for id, sum := range checksums { + result[i] = map[string]string{ + "id": id, + "checksum": sum, + } + i++ + } + return result, nil +} diff --git a/registry.go b/registry.go index 6ae4a425d7..c6179efd6d 100644 --- a/registry.go +++ b/registry.go @@ -1,6 +1,7 @@ package docker import ( + "bytes" "encoding/json" "fmt" "github.com/dotcloud/docker/auth" @@ -315,8 +316,9 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re } // Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, authConfig *auth.AuthConfig) error { +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error { client := graph.getHttpClient() + registry = "https://" + registry + "/v1" // FIXME: Factorize the code // FIXME: Do the puts in goroutines @@ -336,7 +338,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, return err } req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token res, err := client.Do(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -346,7 +348,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, switch res.StatusCode { case 204: // Case where the image is already on the Registry - // FIXME: Do not be silent? + fmt.Fprintf(stdout, "Image %s already uploaded ; skipping.", img.Id) return nil default: errBody, err := ioutil.ReadAll(res.Body) @@ -358,37 +360,26 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, } fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - req2, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", nil) - req2.SetBasicAuth(authConfig.Username, authConfig.Password) - res2, err := client.Do(req2) - if err != nil { - return fmt.Errorf("Registry returned error: %s", err) - } - res2.Body.Close() - if res2.StatusCode != 307 { - return fmt.Errorf("Registry returned unexpected HTTP status code %d, expected 307", res2.StatusCode) - } - url, err := res2.Location() - if err != nil || url == nil { - return fmt.Errorf("Failed to retrieve layer upload location: %s", err) - } - - // FIXME: stream the archive directly to the registry instead of buffering it on disk. This requires either: - // a) Implementing S3's proprietary streaming logic, or - // b) Stream directly to the registry instead of S3. - // I prefer option b. because it doesn't lock us into a proprietary cloud service. - tmpLayer, err := graph.TempLayerArchive(img.Id, Xz, stdout) + layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) + tmp, err := ioutil.ReadAll(layerData2) if err != nil { return err } - defer os.Remove(tmpLayer.Name()) - req3, err := http.NewRequest("PUT", url.String(), ProgressReader(tmpLayer, int(tmpLayer.Size), stdout, "Uploading %v/%v (%v)")) + layerLength := len(tmp) + + layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) + if err != nil { + return fmt.Errorf("Failed to generate layer archive: %s", err) + } + req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", + ProgressReader(layerData.(io.ReadCloser), layerLength, stdout)) if err != nil { return err } req3.ContentLength = int64(tmpLayer.Size) req3.TransferEncoding = []string{"none"} + req3.Header["X-Docker-Token"] = token res3, err := client.Do(req3) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) @@ -406,7 +397,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, // push a tag on the registry. // Remote has the format '/ -func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error { // Keep this for backward compatibility if tag == "" { @@ -415,13 +406,14 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig * // "jsonify" the string revision = "\"" + revision + "\"" + registry = "https://" + registry + "/v1" Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) client := graph.getHttpClient() - req, err := http.NewRequest("PUT", registry+"/users/"+remote+"/"+tag, strings.NewReader(revision)) + req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token res, err := client.Do(req) if err != nil { return err @@ -430,32 +422,25 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig * if res.StatusCode != 200 && res.StatusCode != 201 { return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) } - Debugf("Result of push tag: %d\n", res.StatusCode) - switch res.StatusCode { - default: - return fmt.Errorf("Error %d\n", res.StatusCode) - case 200: - case 201: - } return nil } // FIXME: this should really be PushTag -func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, token []string) error { // Check if the local impage exists img, err := graph.Get(imgId) if err != nil { fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId) return nil } - fmt.Fprintf(stdout, "Pushing tag %s:%s\r\n", remote, tag) + fmt.Fprintf(stdout, "Pushing image %s:%s\r\n", remote, tag) // Push the image - if err = graph.PushImage(stdout, img, registry, authConfig); err != nil { + if err = graph.PushImage(stdout, img, registry, token); err != nil { return err } fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) // And then the tag - if err = graph.pushTag(remote, imgId, registry, tag, authConfig); err != nil { + if err = graph.pushTag(remote, imgId, registry, tag, token); err != nil { return err } return nil @@ -463,19 +448,69 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry // Push a repository to the registry. // Remote has the format '/ -func (graph *Graph) PushRepository(stdout io.Writer, remote, registry string, localRepo Repository, authConfig *auth.AuthConfig) error { - // Check if the remote repository exists/if we have the permission - // if !graph.LookupRemoteRepository(remote, registry, authConfig) { - // return fmt.Errorf("Permission denied on repository %s\n", remote) - // } +func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { + client := graph.getHttpClient() + + checksums, err := graph.Checksums(localRepo) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, nil) + if err != nil { + return err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) + if err != nil { + return err + } + res.Body.Close() + if res.StatusCode != 200 && res.StatusCode != 201 { + return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote) + } + + var token, endpoints []string + if res.Header.Get("X-Docker-Token") != "" { + token = res.Header["X-Docker-Token"] + } else { + return fmt.Errorf("Index response didn't contain an access token") + } + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return fmt.Errorf("Index response didn't contain any endpoints") + } + + for _, registry := range endpoints { + fmt.Fprintf(stdout, "Pushing repository %s to %s (%d tags)\r\n", remote, registry, + len(localRepo)) + // For each image within the repo, push them + for tag, imgId := range localRepo { + if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, token); err != nil { + // FIXME: Continue on error? + return err + } + } + } + checksumsJson, err := json.Marshal(checksums) + if err != nil { + return err + } + req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewBuffer(checksumsJson)) + if err != nil { + return err + } + req2.SetBasicAuth(authConfig.Username, authConfig.Password) + req2.Header["X-Docker-Endpoints"] = endpoints + res2, err := client.Do(req) + if err != nil { + return err + } + res2.Body.Close() + if res2.StatusCode != 204 { + return fmt.Errorf("Error: Status %d trying to push checksums %s", res.StatusCode, remote) + } - // fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) - // // For each image within the repo, push them - // for tag, imgId := range localRepo { - // if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, authConfig); err != nil { - // // FIXME: Continue on error? - // return err - // } - // } return nil } From 6644a3c78a91d2212cc6449f25499e59b42a4397 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 23 Apr 2013 12:02:16 -0700 Subject: [PATCH 08/36] Reactivated CmdPush in commands.go --- commands.go | 103 ++++++++++++++++++++++++++-------------------------- graph.go | 2 +- 2 files changed, 53 insertions(+), 52 deletions(-) diff --git a/commands.go b/commands.go index 8671243f9f..df4746587d 100644 --- a/commands.go +++ b/commands.go @@ -492,64 +492,65 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout rcli.DockerConn, args . return nil } -// func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { -// cmd := rcli.Subcmd(stdout, "push", "NAME", "Push an image or a repository to the registry") -// if err := cmd.Parse(args); err != nil { -// return nil -// } -// local := cmd.Arg(0) +func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { + cmd := rcli.Subcmd(stdout, "push", "NAME", "Push an image or a repository to the registry") + registry := cmd.String("registry", "", "Registry host to push the image to") + if err := cmd.Parse(args); err != nil { + return nil + } + local := cmd.Arg(0) -// if local == "" { -// cmd.Usage() -// return nil -// } + if local == "" { + cmd.Usage() + return nil + } -// // If the login failed, abort -// if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { -// if err := srv.CmdLogin(stdin, stdout, args...); err != nil { -// return err -// } -// if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { -// return fmt.Errorf("Please login prior to push. ('docker login')") -// } -// } + // If the login failed AND we're using the index, abort + if *registry == "" && (srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "") { + if err := srv.CmdLogin(stdin, stdout, args...); err != nil { + return err + } + if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { + return fmt.Errorf("Please login prior to push. ('docker login')") + } + } -// var remote string + var remote string -// tmp := strings.SplitN(local, "/", 2) -// if len(tmp) == 1 { -// return fmt.Errorf( -// "Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", -// srv.runtime.authConfig.Username, local) -// } else { -// remote = local -// } + tmp := strings.SplitN(local, "/", 2) + if len(tmp) == 1 { + return fmt.Errorf( + "Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", + srv.runtime.authConfig.Username, local) + } else { + remote = local + } -// Debugf("Pushing [%s] to [%s]\n", local, remote) + Debugf("Pushing [%s] to [%s]\n", local, remote) -// // Try to get the image -// // FIXME: Handle lookup -// // FIXME: Also push the tags in case of ./docker push myrepo:mytag -// // img, err := srv.runtime.LookupImage(cmd.Arg(0)) -// img, err := srv.runtime.graph.Get(local) -// if err != nil { -// Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) -// // If it fails, try to get the repository -// if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { -// if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { -// return err -// } -// return nil -// } + // Try to get the image + // FIXME: Handle lookup + // FIXME: Also push the tags in case of ./docker push myrepo:mytag + // img, err := srv.runtime.LookupImage(cmd.Arg(0)) + img, err := srv.runtime.graph.Get(local) + if err != nil { + Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) + // If it fails, try to get the repository + if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { + if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { + return err + } + return nil + } -// return err -// } -// err = srv.runtime.graph.PushImage(stdout, img, srv.runtime.authConfig) -// if err != nil { -// return err -// } -// return nil -// } + return err + } + err = srv.runtime.graph.PushImage(stdout, img, *registry, nil) + if err != nil { + return err + } + return nil +} func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "NAME", "Pull an image or a repository from the registry") diff --git a/graph.go b/graph.go index 361a70912f..f6965aa1cf 100644 --- a/graph.go +++ b/graph.go @@ -312,7 +312,7 @@ func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { i := 0 for id, sum := range checksums { result[i] = map[string]string{ - "id": id, + "id": id, "checksum": sum, } i++ From 23953e7d67c83bad8514a262acec5ed20bbbfa94 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 24 Apr 2013 09:11:29 -0700 Subject: [PATCH 09/36] Style changes in auth.Login --- auth/auth.go | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 904198da5f..4696231a93 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -3,7 +3,6 @@ package auth import ( "encoding/base64" "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -16,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const INDEX_SERVER = "https://index.docker.io" +const INDEX_SERVER = "https://indexstaging-docker.dotcloud.com" type AuthConfig struct { Username string `json:"username"` @@ -103,28 +102,24 @@ func Login(authConfig *AuthConfig) (string, error) { storeConfig := false reqStatusCode := 0 var status string - var errMsg string var reqBody []byte jsonBody, err := json.Marshal(authConfig) if err != nil { - errMsg = fmt.Sprintf("Config Error: %s", err) - return "", errors.New(errMsg) + return "", fmt.Errorf("Config Error: %s", err) } // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) req1, err := http.Post(INDEX_SERVER+"/v1/users", "application/json; charset=utf-8", b) if err != nil { - errMsg = fmt.Sprintf("Server Error: %s", err) - return "", errors.New(errMsg) + return "", fmt.Errorf("Server Error: %s", err) } reqStatusCode = req1.StatusCode defer req1.Body.Close() reqBody, err = ioutil.ReadAll(req1.Body) if err != nil { - errMsg = fmt.Sprintf("Server Error: [%#v] %s", reqStatusCode, err) - return "", errors.New(errMsg) + return "", fmt.Errorf("Server Error: [%#v] %s", reqStatusCode, err) } if reqStatusCode == 201 { @@ -149,16 +144,13 @@ func Login(authConfig *AuthConfig) (string, error) { status = "Login Succeeded\n" storeConfig = true } else { - status = fmt.Sprintf("Login: %s", body) - return "", errors.New(status) + return "", fmt.Errorf("Login: %s", body) } } else { - status = fmt.Sprintf("Registration: %s", reqBody) - return "", errors.New(status) + return "", fmt.Errorf("Registration: %s", reqBody) } } else { - status = fmt.Sprintf("[%s] : %s", reqStatusCode, reqBody) - return "", errors.New(status) + return "", fmt.Errorf("Unexpected status code [%d] : %s", reqStatusCode, reqBody) } if storeConfig { authStr := EncodeAuth(authConfig) From 84be35dce10682266b9a35d4156997c84485c769 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 24 Apr 2013 12:15:34 -0700 Subject: [PATCH 10/36] Fixed docker login --- auth/auth.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 4696231a93..2d42667017 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -15,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const INDEX_SERVER = "https://indexstaging-docker.dotcloud.com" +const INDEX_SERVER = "http://indexstaging-docker.dotcloud.com" type AuthConfig struct { Username string `json:"username"` @@ -100,6 +100,7 @@ func saveConfig(rootPath, authStr string, email string) error { // try to register/login to the registry server func Login(authConfig *AuthConfig) (string, error) { storeConfig := false + client := &http.Client{} reqStatusCode := 0 var status string var reqBody []byte @@ -110,11 +111,10 @@ func Login(authConfig *AuthConfig) (string, error) { // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := http.Post(INDEX_SERVER+"/v1/users", "application/json; charset=utf-8", b) + req1, err := http.Post(INDEX_SERVER+"/v1/users/", "application/json; charset=utf-8", b) if err != nil { return "", fmt.Errorf("Server Error: %s", err) } - reqStatusCode = req1.StatusCode defer req1.Body.Close() reqBody, err = ioutil.ReadAll(req1.Body) @@ -127,9 +127,8 @@ func Login(authConfig *AuthConfig) (string, error) { storeConfig = true } else if reqStatusCode == 400 { // FIXME: This should be 'exists', not 'exist'. Need to change on the server first. - if string(reqBody) == "Username or email already exist" { - client := &http.Client{} - req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users", nil) + if string(reqBody) == "\"Username or email already exist\"" { + req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { From 630d358384194489e834d86cb4ae2f6dcc6a0ee5 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 09:15:36 -0700 Subject: [PATCH 11/36] Fixed checksum representation --- container.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container.go b/container.go index 891ae750ae..71686d840d 100644 --- a/container.go +++ b/container.go @@ -2,6 +2,7 @@ package docker import ( "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "github.com/dotcloud/docker/rcli" @@ -705,7 +706,7 @@ func (container *Container) RwChecksum() (string, error) { if _, err := io.Copy(h, rwData); err != nil { return "", err } - return string(h.Sum(nil)), nil + return hex.EncodeToString(h.Sum(nil)), nil } func (container *Container) Export() (Archive, error) { From 4cd9e4722c92f05438779d10bf18206f65417644 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 09:24:29 -0700 Subject: [PATCH 12/36] Fixed graph.Checksums() --- graph.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graph.go b/graph.go index f6965aa1cf..a2a6f47af7 100644 --- a/graph.go +++ b/graph.go @@ -294,8 +294,8 @@ func (graph *Graph) imageRoot(id string) string { } func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { - var checksums map[string]string var result []map[string]string + checksums := map[string]string{} for _, id := range repo { img, err := graph.Get(id) if err != nil { @@ -310,6 +310,7 @@ func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { } } i := 0 + result = make([]map[string]string, len(checksums)) for id, sum := range checksums { result[i] = map[string]string{ "id": id, From 6e936c8fd32357e839cb9337e621b1e152b9fd8b Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 09:36:05 -0700 Subject: [PATCH 13/36] Follow redirections when sending PUT request in PushRepository --- registry.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/registry.go b/registry.go index c6179efd6d..45a9d3bbfc 100644 --- a/registry.go +++ b/registry.go @@ -466,6 +466,20 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return err } res.Body.Close() + for res.StatusCode >= 300 && res.StatusCode < 400 { + Debugf("Redirected to %s\n", res.Header.Get("Location")) + req, err = http.NewRequest("PUT", res.Header.Get("Location"), nil) + if err != nil { + return err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err = client.Do(req) + if err != nil { + return err + } + res.Body.Close() + } + if res.StatusCode != 200 && res.StatusCode != 201 { return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote) } From ea3374bcb0b7a78ccbaeb9cd927fbaf259a5cf29 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 10:10:47 -0700 Subject: [PATCH 14/36] Prepend hash method to the image checksum --- container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container.go b/container.go index 71686d840d..519137ff85 100644 --- a/container.go +++ b/container.go @@ -706,7 +706,7 @@ func (container *Container) RwChecksum() (string, error) { if _, err := io.Copy(h, rwData); err != nil { return "", err } - return hex.EncodeToString(h.Sum(nil)), nil + return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil } func (container *Container) Export() (Archive, error) { From 3c85e9390e2d5fac418e03bac376eb1618487f3c Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 10:12:41 -0700 Subject: [PATCH 15/36] Added X-Docker-Token header to initial index requests --- registry.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/registry.go b/registry.go index 45a9d3bbfc..af69c58adb 100644 --- a/registry.go +++ b/registry.go @@ -239,7 +239,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re client := graph.getHttpClient() fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/checksums" + repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/checksums/" req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -248,6 +248,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re if authConfig != nil { req.SetBasicAuth(authConfig.Username, authConfig.Password) } + req.Header.Set("X-Docker-Token", "true") res, err := client.Do(req) if err != nil { @@ -461,6 +462,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header.Set("X-Docker-Token", "true") res, err := client.Do(req) if err != nil { return err @@ -473,6 +475,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header.Set("X-Docker-Token", "true") res, err = client.Do(req) if err != nil { return err @@ -488,6 +491,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if res.Header.Get("X-Docker-Token") != "" { token = res.Header["X-Docker-Token"] } else { + Debugf("Response headers:\n %s\n", res.Header) return fmt.Errorf("Index response didn't contain an access token") } if res.Header.Get("X-Docker-Endpoints") != "" { From be7560890644759c0ddd229afed208c537f075d7 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 26 Apr 2013 14:00:09 -0700 Subject: [PATCH 16/36] Fixed checksum computing. Ensure checksum is computed when image metadata is loaded from disk. Fixed docker push workflow. Moved hash computing to utils --- container.go | 10 +-- graph.go | 9 ++- image.go | 29 ++++++++ registry.go | 190 ++++++++++++++++++++++++++++----------------------- utils.go | 10 +++ 5 files changed, 155 insertions(+), 93 deletions(-) diff --git a/container.go b/container.go index 519137ff85..5040203227 100644 --- a/container.go +++ b/container.go @@ -1,8 +1,6 @@ package docker import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "github.com/dotcloud/docker/rcli" @@ -698,15 +696,11 @@ func (container *Container) ExportRw() (Archive, error) { } func (container *Container) RwChecksum() (string, error) { - h := sha256.New() - rwData, err := container.ExportRw() + rwData, err := Tar(container.rwPath(), Xz) if err != nil { return "", err } - if _, err := io.Copy(h, rwData); err != nil { - return "", err - } - return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil + return HashData(rwData) } func (container *Container) Export() (Archive, error) { diff --git a/graph.go b/graph.go index a2a6f47af7..d7a53be7c3 100644 --- a/graph.go +++ b/graph.go @@ -81,6 +81,13 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) } img.graph = graph + + if img.Checksum == "" { + err := img.FixChecksum() + if err != nil { + return nil, err + } + } return img, nil } @@ -98,7 +105,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Parent = container.Image img.Container = container.Id img.ContainerConfig = *container.Config - // FIXME: If an image is pulled from a raw URL (not created from a container), + // FIXME: If an image is imported from a raw URL (not created from a container), // its checksum will not be computed, which will cause a push to fail checksum, err := container.RwChecksum() if err != nil { diff --git a/image.go b/image.go index 8090e583e6..4d625d6c0f 100644 --- a/image.go +++ b/image.go @@ -52,6 +52,7 @@ func LoadImage(root string) (*Image, error) { } else if !stat.IsDir() { return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root)) } + return &img, nil } @@ -258,3 +259,31 @@ func (img *Image) layer() (string, error) { } return layerPath(root), nil } + +func (img *Image) FixChecksum() error { + layer, err := img.layer() + if err != nil { + return err + } + layerData, err := Tar(layer, Xz) + if err != nil { + return err + } + sum, err := HashData(layerData) + if err != nil { + return err + } + img.Checksum = sum + jsonData, err := json.Marshal(img) + if err != nil { + return err + } + root, err := img.root() + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + return nil +} diff --git a/registry.go b/registry.go index af69c58adb..d9850843d1 100644 --- a/registry.go +++ b/registry.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/dotcloud/docker/auth" + "github.com/shin-/cookiejar" "io" "io/ioutil" "net/http" @@ -14,7 +15,6 @@ import ( ) //FIXME: Set the endpoint in a conf file or via commandline -//const INDEX_ENDPOINT = "http://registry-creack.dotcloud.com/v1" const INDEX_ENDPOINT = auth.INDEX_SERVER + "/v1" // Build an Image object from raw json data @@ -47,6 +47,13 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { return ret, nil } +func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { + for _, cookie := range c.Jar.Cookies(req.URL) { + req.AddCookie(cookie) + } + return c.Do(req) +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) { @@ -57,7 +64,6 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] return nil, err } req.Header["X-Docker-Token"] = token - // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -82,7 +88,9 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] func (graph *Graph) getHttpClient() *http.Client { if graph.httpClient == nil { graph.httpClient = new(http.Client) + graph.httpClient.Jar = cookiejar.NewCookieJar() } + Debugf("cookies: %v",graph.httpClient.Jar) return graph.httpClient } @@ -111,7 +119,6 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok return nil, nil, fmt.Errorf("Failed to download json: %s", err) } req.Header["X-Docker-Token"] = token - // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -139,7 +146,6 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header["X-Docker-Token"] = token - // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -271,7 +277,6 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return fmt.Errorf("Index response didn't contain any endpoints") } - // FIXME: If askedTag is empty, fetch all tags. var tagsList map[string]string if askedTag == "" { tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) @@ -316,86 +321,86 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return nil } -// Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error { +func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, token []string) error { + if parent, err := img.GetParent(); err != nil { + return err + } else if parent != nil { + if err := pushImageRec(graph, stdout, parent, registry, token); err != nil { + return err + } + } client := graph.getHttpClient() - registry = "https://" + registry + "/v1" + jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) + if err != nil { + return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) + } - // FIXME: Factorize the code - // FIXME: Do the puts in goroutines - if err := imgOrig.WalkHistory(func(img *Image) error { + fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) - jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) - if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) - } - - fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) - - // FIXME: try json with UTF8 - jsonData := strings.NewReader(string(jsonRaw)) - req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - req.Header["X-Docker-Token"] = token - res, err := client.Do(req) - if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - switch res.StatusCode { - case 204: - // Case where the image is already on the Registry - fmt.Fprintf(stdout, "Image %s already uploaded ; skipping.", img.Id) - return nil - default: - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - errBody = []byte(err.Error()) - } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) - } - } - - fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) - tmp, err := ioutil.ReadAll(layerData2) - if err != nil { - return err - } - layerLength := len(tmp) - - layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) - if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) - } - req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", - ProgressReader(layerData.(io.ReadCloser), layerLength, stdout)) - if err != nil { - return err - } - req3.ContentLength = int64(tmpLayer.Size) - - req3.TransferEncoding = []string{"none"} - req3.Header["X-Docker-Token"] = token - res3, err := client.Do(req3) - if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) - } - res3.Body.Close() - if res3.StatusCode != 200 { - return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) - } - return nil - }); err != nil { + // FIXME: try json with UTF8 + jsonData := strings.NewReader(string(jsonRaw)) + req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) + if err != nil { return err } + req.Header.Add("Content-type", "application/json") + req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + res, err := doWithCookies(client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + errBody = []byte(err.Error()) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + fmt.Fprintf(stdout, "Image %v already uploaded ; skipping\n", img.Id) + return nil + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + + fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) + + layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) + if err != nil { + return fmt.Errorf("Failed to generate layer archive: %s", err) + } + req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", + layerData) + if err != nil { + return err + } + + req3.ContentLength = -1 + req3.TransferEncoding = []string{"chunked"} + req3.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + fmt.Printf("%v", req3.Header) + res3, err := doWithCookies(client, req3) + if err != nil { + return fmt.Errorf("Failed to upload layer: %s", err) + } + res3.Body.Close() + if res3.StatusCode != 200 { + return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) + } return nil } +// Push a local image to the registry with its history if needed +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error { + registry = "https://" + registry + "/v1" + return pushImageRec(graph, stdout, imgOrig, registry, token) +} + // push a tag on the registry. // Remote has the format '/ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error { @@ -413,9 +418,13 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []stri client := graph.getHttpClient() req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + if err != nil { + return err + } req.Header.Add("Content-type", "application/json") - req.Header["X-Docker-Token"] = token - res, err := client.Do(req) + req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + req.ContentLength = int64(len(revision)) + res, err := doWithCookies(client, req) if err != nil { return err } @@ -441,7 +450,7 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry } fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) // And then the tag - if err = graph.pushTag(remote, imgId, registry, tag, token); err != nil { + if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil { return err } return nil @@ -453,15 +462,25 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re client := graph.getHttpClient() checksums, err := graph.Checksums(localRepo) + imgList := make([]map[string]string, len(checksums)) if err != nil { return err } - req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, nil) + for i, obj := range checksums { + imgList[i] = map[string]string{"id": obj["id"]} + } + imgListJson, err := json.Marshal(imgList) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, bytes.NewReader(imgListJson)) if err != nil { return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.ContentLength = int64(len(imgListJson)) req.Header.Set("X-Docker-Token", "true") res, err := client.Do(req) if err != nil { @@ -470,11 +489,12 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re res.Body.Close() for res.StatusCode >= 300 && res.StatusCode < 400 { Debugf("Redirected to %s\n", res.Header.Get("Location")) - req, err = http.NewRequest("PUT", res.Header.Get("Location"), nil) + req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) if err != nil { return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.ContentLength = int64(len(imgListJson)) req.Header.Set("X-Docker-Token", "true") res, err = client.Do(req) if err != nil { @@ -490,6 +510,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re var token, endpoints []string if res.Header.Get("X-Docker-Token") != "" { token = res.Header["X-Docker-Token"] + Debugf("Auth token: %v", token) } else { Debugf("Response headers:\n %s\n", res.Header) return fmt.Errorf("Index response didn't contain an access token") @@ -515,13 +536,14 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if err != nil { return err } - req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewBuffer(checksumsJson)) + req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(checksumsJson)) if err != nil { return err } req2.SetBasicAuth(authConfig.Username, authConfig.Password) req2.Header["X-Docker-Endpoints"] = endpoints - res2, err := client.Do(req) + req2.ContentLength = int64(len(checksumsJson)) + res2, err := client.Do(req2) if err != nil { return err } diff --git a/utils.go b/utils.go index 297b798af8..bcd3e0d1b3 100644 --- a/utils.go +++ b/utils.go @@ -2,6 +2,8 @@ package docker import ( "bytes" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "github.com/dotcloud/docker/rcli" @@ -456,3 +458,11 @@ func FindCgroupMountpoint(cgroupType string) (string, error) { return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) } + +func HashData(src io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil +} \ No newline at end of file From 1c76f91fc47637efff99ab7e5c5be8f4f84facb0 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 29 Apr 2013 06:56:33 -0700 Subject: [PATCH 17/36] Fixed minor bugs in docker pull --- registry.go | 51 ++++++++++++++++----------------------------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/registry.go b/registry.go index d9850843d1..5da811b123 100644 --- a/registry.go +++ b/registry.go @@ -29,24 +29,6 @@ func NewImgJson(src []byte) (*Image, error) { return ret, nil } -// Build an Image object list from a raw json data -// FIXME: Do this in "stream" mode -func NewMultipleImgJson(src []byte) ([]*Image, error) { - ret := []*Image{} - - dec := json.NewDecoder(strings.NewReader(string(src))) - for { - m := &Image{} - if err := dec.Decode(m); err == io.EOF { - break - } else if err != nil { - return nil, err - } - ret = append(ret, m) - } - return ret, nil -} - func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -56,14 +38,14 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) { +func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]string, error) { client := graph.getHttpClient() req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -78,11 +60,12 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] return nil, fmt.Errorf("Error while reading the http response: %s\n", err) } - history, err := NewMultipleImgJson(jsonString) - if err != nil { - return nil, fmt.Errorf("Error while parsing the json: %s\n", err) + Debugf("Ancestry: %s", jsonString) + history := new([]string) + if err := json.Unmarshal(jsonString, history); err != nil { + return nil, err } - return history, nil + return *history, nil } func (graph *Graph) getHttpClient() *http.Client { @@ -90,7 +73,6 @@ func (graph *Graph) getHttpClient() *http.Client { graph.httpClient = new(http.Client) graph.httpClient.Jar = cookiejar.NewCookieJar() } - Debugf("cookies: %v",graph.httpClient.Jar) return graph.httpClient } @@ -118,7 +100,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -145,7 +127,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -161,7 +143,7 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit if err != nil { return nil, err } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err := client.Do(req) defer res.Body.Close() if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { @@ -171,7 +153,7 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit return nil, fmt.Errorf("Repository not found") } - var result *map[string]string + result := new(map[string]string) rawJson, err := ioutil.ReadAll(res.Body) if err != nil { @@ -196,7 +178,7 @@ func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry strin if err != nil { return "", err } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { return "", fmt.Errorf("Error while retrieving repository info: %v", err) @@ -226,9 +208,9 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token [] } // FIXME: Try to stream the images? // FIXME: Launch the getRemoteImage() in goroutines - for _, j := range history { - if !graph.Exists(j.Id) { - img, layer, err := graph.getRemoteImage(stdout, j.Id, registry, token) + for _, id := range history { + if !graph.Exists(id) { + img, layer, err := graph.getRemoteImage(stdout, id, registry, token) if err != nil { // FIXME: Keep goging in case of error? return err @@ -245,7 +227,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re client := graph.getHttpClient() fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/checksums/" + repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/images" req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -512,7 +494,6 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re token = res.Header["X-Docker-Token"] Debugf("Auth token: %v", token) } else { - Debugf("Response headers:\n %s\n", res.Header) return fmt.Errorf("Index response didn't contain an access token") } if res.Header.Get("X-Docker-Endpoints") != "" { From e81a53eea9d5a44c3fde8d3060845ca18e95b4c4 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 30 Apr 2013 06:55:24 -0700 Subject: [PATCH 18/36] Added support for REPO:TAG format in docker pull (overrides -t option) --- commands.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index df4746587d..241923cef7 100644 --- a/commands.go +++ b/commands.go @@ -565,6 +565,12 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } + if strings.Contains(remote, ":") { + remoteParts := strings.Split(remote, ":") + tag = &remoteParts[1] + remote = remoteParts[0] + } + // FIXME: CmdPull should be a wrapper around Runtime.Pull() if *registry != "" { if err := srv.runtime.graph.PullImage(stdout, remote, *registry, nil); err != nil { @@ -572,7 +578,6 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string } return nil } - // FIXME: Allow pull repo:tag if err := srv.runtime.graph.PullRepository(stdout, remote, *tag, srv.runtime.repositories, srv.runtime.authConfig); err != nil { return err } From 6e2ddf6f608aa3d73b26b22dc3ff27f11534cdee Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 30 Apr 2013 10:01:19 -0700 Subject: [PATCH 19/36] Checksum system overhaul --- commands.go | 3 --- graph.go | 20 +++++------------- image.go | 60 +++++++++++++++++++++++++++++++++++++++-------------- registry.go | 14 ++++++++++--- 4 files changed, 60 insertions(+), 37 deletions(-) diff --git a/commands.go b/commands.go index 241923cef7..06443b559e 100644 --- a/commands.go +++ b/commands.go @@ -529,9 +529,6 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ... Debugf("Pushing [%s] to [%s]\n", local, remote) // Try to get the image - // FIXME: Handle lookup - // FIXME: Also push the tags in case of ./docker push myrepo:mytag - // img, err := srv.runtime.LookupImage(cmd.Arg(0)) img, err := srv.runtime.graph.Get(local) if err != nil { Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) diff --git a/graph.go b/graph.go index d7a53be7c3..6e443f206b 100644 --- a/graph.go +++ b/graph.go @@ -81,13 +81,6 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) } img.graph = graph - - if img.Checksum == "" { - err := img.FixChecksum() - if err != nil { - return nil, err - } - } return img, nil } @@ -105,17 +98,11 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Parent = container.Image img.Container = container.Id img.ContainerConfig = *container.Config - // FIXME: If an image is imported from a raw URL (not created from a container), - // its checksum will not be computed, which will cause a push to fail - checksum, err := container.RwChecksum() - if err != nil { - return nil, err - } - img.Checksum = checksum } if err := graph.Register(layerData, img); err != nil { return nil, err } + img.Checksum() return img, nil } @@ -309,7 +296,10 @@ func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { return nil, err } err = img.WalkHistory(func(image *Image) error { - checksums[image.Id] = image.Checksum + checksums[image.Id], err = image.Checksum() + if err != nil { + return err + } return nil }) if err != nil { diff --git a/image.go b/image.go index 4d625d6c0f..dcad130d3b 100644 --- a/image.go +++ b/image.go @@ -1,7 +1,9 @@ package docker import ( + "bytes" "crypto/rand" + "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -18,7 +20,6 @@ import ( type Image struct { Id string `json:"id"` Parent string `json:"parent,omitempty"` - Checksum string `json:"checksum,omitempty"` Comment string `json:"comment,omitempty"` Created time.Time `json:"created"` Container string `json:"container,omitempty"` @@ -260,30 +261,57 @@ func (img *Image) layer() (string, error) { return layerPath(root), nil } -func (img *Image) FixChecksum() error { +func (img *Image) Checksum() (string, error) { + root, err := img.root() + if err != nil { + return "", err + } + + checksumDictPth := path.Join(root, "..", "..", "checksums") + checksums := new(map[string]string) + + if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { + if err := json.Unmarshal(checksumDict, checksums); err != nil { + return "", err + } + if checksum, ok := (*checksums)[img.Id]; ok { + return checksum, nil + } + } + layer, err := img.layer() if err != nil { - return err + return "", err } layerData, err := Tar(layer, Xz) if err != nil { - return err + return "", err } - sum, err := HashData(layerData) + h := sha256.New() + if _, err := io.Copy(h, layerData); err != nil { + return "", err + } + + jsonData, err := ioutil.ReadFile(jsonPath(root)) if err != nil { - return err + return "", err } - img.Checksum = sum - jsonData, err := json.Marshal(img) + if _, err := io.Copy(h, bytes.NewBuffer(jsonData)); err != nil { + return "", err + } + + hash := "sha256:"+hex.EncodeToString(h.Sum(nil)) + if *checksums == nil { + *checksums = map[string]string{} + } + (*checksums)[img.Id] = hash + checksumJson, err := json.Marshal(checksums) if err != nil { - return err + return hash, err } - root, err := img.root() - if err != nil { - return err + + if err := ioutil.WriteFile(checksumDictPth, checksumJson, 0600); err != nil { + return hash, err } - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err - } - return nil + return hash, nil } diff --git a/registry.go b/registry.go index 5da811b123..be9d37edc1 100644 --- a/registry.go +++ b/registry.go @@ -327,6 +327,12 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + + checksum, err := img.Checksum() + if err != nil { + return fmt.Errorf("Error while retrieving checksum for %s: %v", img.Id, err) + } + req.Header.Set("X-Docker-Checksum", checksum) res, err := doWithCookies(client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -457,7 +463,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return err } - req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, bytes.NewReader(imgListJson)) + req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/", bytes.NewReader(imgListJson)) if err != nil { return err } @@ -468,7 +474,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if err != nil { return err } - res.Body.Close() + defer res.Body.Close() for res.StatusCode >= 300 && res.StatusCode < 400 { Debugf("Redirected to %s\n", res.Header.Get("Location")) req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) @@ -482,10 +488,12 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if err != nil { return err } - res.Body.Close() + defer res.Body.Close() } if res.StatusCode != 200 && res.StatusCode != 201 { + info, err := ioutil.ReadAll(res.Body) + Debugf("%v %v", err, string(info)) return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote) } From f10b0f75e02041db64176760a3be9639408ad923 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 30 Apr 2013 14:05:33 -0700 Subject: [PATCH 20/36] Fix checksum computing --- image.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/image.go b/image.go index dcad130d3b..2142716d4f 100644 --- a/image.go +++ b/image.go @@ -283,22 +283,26 @@ func (img *Image) Checksum() (string, error) { if err != nil { return "", err } - layerData, err := Tar(layer, Xz) - if err != nil { - return "", err - } - h := sha256.New() - if _, err := io.Copy(h, layerData); err != nil { - return "", err - } - jsonData, err := ioutil.ReadFile(jsonPath(root)) if err != nil { return "", err } + + layerData, err := Tar(layer, Xz) + if err != nil { + return "", err + } + + h := sha256.New() if _, err := io.Copy(h, bytes.NewBuffer(jsonData)); err != nil { return "", err } + if _, err := io.Copy(h, strings.NewReader("\n")); err != nil { + return "", err + } + if _, err := io.Copy(h, layerData); err != nil { + return "", err + } hash := "sha256:"+hex.EncodeToString(h.Sum(nil)) if *checksums == nil { From b5873806d0f155b0fc16d224427f90548bfa8afe Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 30 Apr 2013 14:11:16 -0700 Subject: [PATCH 21/36] Only send checksums for images not uploaded yet --- registry.go | 62 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/registry.go b/registry.go index be9d37edc1..d44a6932b1 100644 --- a/registry.go +++ b/registry.go @@ -89,6 +89,37 @@ func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.A return err == nil && res.StatusCode == 307 } +func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { + u := INDEX_ENDPOINT+"/repositories/"+repository+"/images" + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := graph.getHttpClient().Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // Repository doesn't exist yet + if res.StatusCode == 404 { + return nil, nil + } + + jsonData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + imageList := []map[string]string{} + err = json.Unmarshal(jsonData, &imageList) + if err != nil { + return nil, err + } + + return imageList, nil +} + // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, token []string) (*Image, Archive, error) { @@ -362,6 +393,7 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t if err != nil { return fmt.Errorf("Failed to generate layer archive: %s", err) } + req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", layerData) if err != nil { @@ -451,13 +483,36 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re checksums, err := graph.Checksums(localRepo) imgList := make([]map[string]string, len(checksums)) + checksums2 := make([]map[string]string, len(checksums)) if err != nil { return err } - for i, obj := range checksums { - imgList[i] = map[string]string{"id": obj["id"]} + uploadedImages, err := graph.getImagesInRepository(remote, authConfig) + if err != nil { + return fmt.Errorf("Error occured while fetching the list") } + + + // Filter list to only send images/checksums not already uploaded + i := 0 + for _, obj := range checksums { + found := false + for _, uploadedImg := range uploadedImages { + if obj["id"] == uploadedImg["id"] && uploadedImg["checksum"] != "" { + found = true + break + } + } + if !found { + imgList[i] = map[string]string{"id": obj["id"]} + checksums2[i] = obj + i += 1 + } + } + checksums = checksums2[:i] + imgList = imgList[:i] + imgListJson, err := json.Marshal(imgList) if err != nil { return err @@ -492,8 +547,6 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re } if res.StatusCode != 200 && res.StatusCode != 201 { - info, err := ioutil.ReadAll(res.Body) - Debugf("%v %v", err, string(info)) return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote) } @@ -525,6 +578,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if err != nil { return err } + req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(checksumsJson)) if err != nil { return err From 19045b530eeacc97941cc1b63c2626f92df10f67 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 05:08:27 -0700 Subject: [PATCH 22/36] simplify graph.Checksums --- graph.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/graph.go b/graph.go index 6e443f206b..f51597df9a 100644 --- a/graph.go +++ b/graph.go @@ -297,10 +297,7 @@ func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { } err = img.WalkHistory(func(image *Image) error { checksums[image.Id], err = image.Checksum() - if err != nil { - return err - } - return nil + return err }) if err != nil { return nil, err From be791a223b65d5641e032ea78ad7289b6eb58979 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 05:11:06 -0700 Subject: [PATCH 23/36] simplify image.Checksum --- image.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/image.go b/image.go index 2142716d4f..583aad0ba8 100644 --- a/image.go +++ b/image.go @@ -1,7 +1,6 @@ package docker import ( - "bytes" "crypto/rand" "crypto/sha256" "encoding/hex" @@ -294,10 +293,10 @@ func (img *Image) Checksum() (string, error) { } h := sha256.New() - if _, err := io.Copy(h, bytes.NewBuffer(jsonData)); err != nil { + if _, err := h.Write(jsonData); err != nil { return "", err } - if _, err := io.Copy(h, strings.NewReader("\n")); err != nil { + if _, err := h.Write([]byte("\n")); err != nil { return "", err } if _, err := io.Copy(h, layerData); err != nil { From 5690562fc855a7e813a7e3cfb3340f60bd626530 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 05:26:52 -0700 Subject: [PATCH 24/36] Fix error in PushImage --- registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/registry.go b/registry.go index d44a6932b1..e8fcd1fd11 100644 --- a/registry.go +++ b/registry.go @@ -375,7 +375,8 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - errBody = []byte(err.Error()) + return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ + " trying to parse response body: %v", res.StatusCode, err) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { From 594827d41640eb6a6a878c29aca87f7a4ce6249c Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 09:06:17 -0700 Subject: [PATCH 25/36] Fixed typo in 'username or email already exists' --- auth/auth.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 2d42667017..a653f5563e 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -126,8 +126,7 @@ func Login(authConfig *AuthConfig) (string, error) { status = "Account Created\n" storeConfig = true } else if reqStatusCode == 400 { - // FIXME: This should be 'exists', not 'exist'. Need to change on the server first. - if string(reqBody) == "\"Username or email already exist\"" { + if string(reqBody) == "\"Username or email already exists\"" { req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) From 18796d55a66f2f072349c54085516fe038bd8283 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 13:41:58 -0700 Subject: [PATCH 26/36] Fixed some login quirks --- auth/auth.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index a653f5563e..f8e94884dd 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -15,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const INDEX_SERVER = "http://indexstaging-docker.dotcloud.com" +const INDEX_SERVER = "https://indexstaging-docker.dotcloud.com" type AuthConfig struct { Username string `json:"username"` @@ -123,8 +123,12 @@ func Login(authConfig *AuthConfig) (string, error) { } if reqStatusCode == 201 { - status = "Account Created\n" + status = "Account created. Please use the confirmation link we sent"+ + " to your e-mail to activate it.\n" storeConfig = true + } else if reqStatusCode == 403 { + return "", fmt.Errorf("Login: Your account hasn't been activated. "+ + "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users/", nil) From 0f68042053ac652de13cd68086d0188ac0ecf9b6 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 1 May 2013 16:36:01 -0700 Subject: [PATCH 27/36] Handled wrong user credentials by re-init the auth file (it was impossible to login after having wrong crendentials) --- auth/auth.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index f8e94884dd..b06086ff6a 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -75,6 +75,9 @@ func LoadConfig(rootPath string) (*AuthConfig, error) { return nil, err } arr := strings.Split(string(b), "\n") + if len(arr) < 2 { + return nil, fmt.Errorf("The Auth config file is empty") + } origAuth := strings.Split(arr[0], " = ") origEmail := strings.Split(arr[1], " = ") authConfig, err := DecodeAuth(origAuth[1]) @@ -88,9 +91,14 @@ func LoadConfig(rootPath string) (*AuthConfig, error) { // save the auth config func saveConfig(rootPath, authStr string, email string) error { + confFile := path.Join(rootPath, CONFIGFILE) + if len(email) == 0 { + os.Remove(confFile) + return nil + } lines := "auth = " + authStr + "\n" + "email = " + email + "\n" b := []byte(lines) - err := ioutil.WriteFile(path.Join(rootPath, CONFIGFILE), b, 0600) + err := ioutil.WriteFile(confFile, b, 0600) if err != nil { return err } @@ -145,8 +153,12 @@ func Login(authConfig *AuthConfig) (string, error) { if resp.StatusCode == 200 { status = "Login Succeeded\n" storeConfig = true + } else if resp.StatusCode == 401 { + saveConfig(authConfig.rootPath, "", "") + return "", fmt.Errorf("Wrong login/password, please try again") } else { - return "", fmt.Errorf("Login: %s", body) + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, + resp.StatusCode, resp.Header) } } else { return "", fmt.Errorf("Registration: %s", reqBody) From 0c5e76958bbfd5c4d315d75262381cbae5c5f048 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 2 May 2013 08:06:11 -0700 Subject: [PATCH 28/36] Use progress reader when uploading/downloading layers --- registry.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/registry.go b/registry.go index e8fcd1fd11..b5fd734bcd 100644 --- a/registry.go +++ b/registry.go @@ -390,13 +390,13 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) + layerData, err := graph.TempLayerArchive(img.Id, Xz, stdout) if err != nil { return fmt.Errorf("Failed to generate layer archive: %s", err) } req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", - layerData) + ProgressReader(layerData, -1, stdout, "")) if err != nil { return err } @@ -404,7 +404,6 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t req3.ContentLength = -1 req3.TransferEncoding = []string{"chunked"} req3.Header.Set("Authorization", "Token " + strings.Join(token, ",")) - fmt.Printf("%v", req3.Header) res3, err := doWithCookies(client, req3) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) @@ -491,7 +490,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re uploadedImages, err := graph.getImagesInRepository(remote, authConfig) if err != nil { - return fmt.Errorf("Error occured while fetching the list") + return fmt.Errorf("Error occured while fetching the list: %v", err) } From d985050aeb3f1411b94b21f6db35a24f9a1f93f2 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 2 May 2013 08:18:33 -0700 Subject: [PATCH 29/36] gofmt pass --- image.go | 2 +- registry.go | 19 +++++++++---------- utils.go | 17 +++++++++-------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/image.go b/image.go index 583aad0ba8..bf86e2e7f7 100644 --- a/image.go +++ b/image.go @@ -303,7 +303,7 @@ func (img *Image) Checksum() (string, error) { return "", err } - hash := "sha256:"+hex.EncodeToString(h.Sum(nil)) + hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) if *checksums == nil { *checksums = map[string]string{} } diff --git a/registry.go b/registry.go index b5fd734bcd..0aab3f003e 100644 --- a/registry.go +++ b/registry.go @@ -45,7 +45,7 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] if err != nil { return nil, err } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -90,7 +90,7 @@ func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.A } func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { - u := INDEX_ENDPOINT+"/repositories/"+repository+"/images" + u := INDEX_ENDPOINT + "/repositories/" + repository + "/images" req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err @@ -131,7 +131,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -158,7 +158,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -174,7 +174,7 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit if err != nil { return nil, err } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) defer res.Body.Close() if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { @@ -209,7 +209,7 @@ func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry strin if err != nil { return "", err } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { return "", fmt.Errorf("Error while retrieving repository info: %v", err) @@ -357,7 +357,7 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t return err } req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) checksum, err := img.Checksum() if err != nil { @@ -403,7 +403,7 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t req3.ContentLength = -1 req3.TransferEncoding = []string{"chunked"} - req3.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + req3.Header.Set("Authorization", "Token "+strings.Join(token, ",")) res3, err := doWithCookies(client, req3) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) @@ -442,7 +442,7 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []stri return err } req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) req.ContentLength = int64(len(revision)) res, err := doWithCookies(client, req) if err != nil { @@ -493,7 +493,6 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return fmt.Errorf("Error occured while fetching the list: %v", err) } - // Filter list to only send images/checksums not already uploaded i := 0 for _, obj := range checksums { diff --git a/utils.go b/utils.go index bcd3e0d1b3..78e13ed881 100644 --- a/utils.go +++ b/utils.go @@ -397,6 +397,15 @@ func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) return written, err } + +func HashData(src io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + type KernelVersionInfo struct { Kernel int Major int @@ -457,12 +466,4 @@ func FindCgroupMountpoint(cgroupType string) (string, error) { } return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) -} - -func HashData(src io.Reader) (string, error) { - h := sha256.New() - if _, err := io.Copy(h, src); err != nil { - return "", err - } - return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil } \ No newline at end of file From a372f982c1383a3b4d742ef4f4de1a645cd7434d Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 2 May 2013 15:39:44 -0700 Subject: [PATCH 30/36] Switching to prod index server --- auth/auth.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index b06086ff6a..5a5987ace8 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -15,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const INDEX_SERVER = "https://indexstaging-docker.dotcloud.com" +const INDEX_SERVER = "https://index.docker.io" type AuthConfig struct { Username string `json:"username"` @@ -131,11 +131,11 @@ func Login(authConfig *AuthConfig) (string, error) { } if reqStatusCode == 201 { - status = "Account created. Please use the confirmation link we sent"+ + status = "Account created. Please use the confirmation link we sent" + " to your e-mail to activate it.\n" storeConfig = true } else if reqStatusCode == 403 { - return "", fmt.Errorf("Login: Your account hasn't been activated. "+ + return "", fmt.Errorf("Login: Your account hasn't been activated. " + "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { From bcdf03037b5f11d3760b45c96b9180f7b8da4b40 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 2 May 2013 18:52:23 -0700 Subject: [PATCH 31/36] Fixed pulling repositories from library --- registry.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/registry.go b/registry.go index 0aab3f003e..592b13da6e 100644 --- a/registry.go +++ b/registry.go @@ -168,6 +168,11 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { client := graph.getHttpClient() + if strings.Count(repository, "/") == 0 { + // This will be removed once the Registry supports auto-resolution on + // the "library" namespace + repository = "library/" + repository + } for _, host := range registries { endpoint := "https://" + host + "/v1/repositories/" + repository + "/tags" req, err := http.NewRequest("GET", endpoint, nil) @@ -257,7 +262,7 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token [] func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := graph.getHttpClient() - fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) + fmt.Fprintf(stdout, "Pulling repository %s from %s\r\n", remote, INDEX_ENDPOINT) repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/images" req, err := http.NewRequest("GET", repositoryTarget, nil) From 3febeb93f553be0dceec81591a6e0ba578b42a12 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 2 May 2013 18:57:15 -0700 Subject: [PATCH 32/36] Added help message to invite to login when getting a 401 --- registry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/registry.go b/registry.go index 592b13da6e..a471687609 100644 --- a/registry.go +++ b/registry.go @@ -279,6 +279,9 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return err } defer res.Body.Close() + if res.StatusCode == 401 { + return fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode != 200 { From 00266df8ac64bb3b7e19818ad2dfb852dc0d2e7f Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 3 May 2013 15:37:28 -0700 Subject: [PATCH 33/36] Fixed public pull + Added some verbosity about what is happening --- registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/registry.go b/registry.go index a471687609..43738646db 100644 --- a/registry.go +++ b/registry.go @@ -269,7 +269,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re if err != nil { return err } - if authConfig != nil { + if authConfig != nil && len(authConfig.Username) > 0 { req.SetBasicAuth(authConfig.Username, authConfig.Password) } req.Header.Set("X-Docker-Token", "true") @@ -309,6 +309,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re } for askedTag, imgId := range tagsList { + fmt.Fprintf(stdout, "Resolving tag \"%s:%s\" from %s\n", remote, askedTag, endpoints) success := false for _, registry := range endpoints { if imgId == "" { From b0e076f374c8919ba29ec24a8cd27785a8e0827c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 15:30:03 -0700 Subject: [PATCH 34/36] Add output to checksums, code cleaning --- graph.go | 28 -------------------------- registry.go | 58 ++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/graph.go b/graph.go index f51597df9a..853f8c736b 100644 --- a/graph.go +++ b/graph.go @@ -286,31 +286,3 @@ func (graph *Graph) Heads() (map[string]*Image, error) { func (graph *Graph) imageRoot(id string) string { return path.Join(graph.Root, id) } - -func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { - var result []map[string]string - checksums := map[string]string{} - for _, id := range repo { - img, err := graph.Get(id) - if err != nil { - return nil, err - } - err = img.WalkHistory(func(image *Image) error { - checksums[image.Id], err = image.Checksum() - return err - }) - if err != nil { - return nil, err - } - } - i := 0 - result = make([]map[string]string, len(checksums)) - for id, sum := range checksums { - result[i] = map[string]string{ - "id": id, - "checksum": sum, - } - i++ - } - return result, nil -} diff --git a/registry.go b/registry.go index 43738646db..b0faee3e71 100644 --- a/registry.go +++ b/registry.go @@ -95,7 +95,9 @@ func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.Au if err != nil { return nil, err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + if authConfig != nil && len(authConfig.Username) > 0 { + req.SetBasicAuth(authConfig.Username, authConfig.Password) + } res, err := graph.getHttpClient().Do(req) if err != nil { return nil, err @@ -111,9 +113,12 @@ func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.Au if err != nil { return nil, err } + imageList := []map[string]string{} + err = json.Unmarshal(jsonData, &imageList) if err != nil { + Debugf("Body: %s (%s)\n", res.Body, u) return nil, err } @@ -174,7 +179,7 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit repository = "library/" + repository } for _, host := range registries { - endpoint := "https://" + host + "/v1/repositories/" + repository + "/tags" + endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -433,12 +438,6 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, // push a tag on the registry. // Remote has the format '/ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error { - - // Keep this for backward compatibility - if tag == "" { - tag = "lastest" - } - // "jsonify" the string revision = "\"" + revision + "\"" registry = "https://" + registry + "/v1" @@ -490,16 +489,17 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { client := graph.getHttpClient() - checksums, err := graph.Checksums(localRepo) - imgList := make([]map[string]string, len(checksums)) - checksums2 := make([]map[string]string, len(checksums)) + checksums, err := graph.Checksums(stdout, localRepo) if err != nil { return err } + imgList := make([]map[string]string, len(checksums)) + checksums2 := make([]map[string]string, len(checksums)) + uploadedImages, err := graph.getImagesInRepository(remote, authConfig) if err != nil { - return fmt.Errorf("Error occured while fetching the list: %v", err) + return fmt.Errorf("Error occured while fetching the list: %s", err) } // Filter list to only send images/checksums not already uploaded @@ -605,3 +605,37 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return nil } + +func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]string, error) { + var result []map[string]string + checksums := map[string]string{} + for _, id := range repo { + img, err := graph.Get(id) + if err != nil { + return nil, err + } + err = img.WalkHistory(func(image *Image) error { + fmt.Fprintf(output, "Computing checksum for image %s\n", image.Id) + if _, exists := checksums[image.Id]; !exists { + checksums[image.Id], err = image.Checksum() + if err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, err + } + } + i := 0 + result = make([]map[string]string, len(checksums)) + for id, sum := range checksums { + result[i] = map[string]string{ + "id": id, + "checksum": sum, + } + i++ + } + return result, nil +} From c9994ed0fb01c65304fa8bd2eaf80f8a5edd6e41 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Sat, 4 May 2013 09:22:45 -0700 Subject: [PATCH 35/36] Moved the Debugf message in a registry to a more useful place --- registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry.go b/registry.go index b0faee3e71..72379dd3dd 100644 --- a/registry.go +++ b/registry.go @@ -187,8 +187,8 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) defer res.Body.Close() + Debugf("Got status code %d from %s", res.StatusCode, endpoint) if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { - Debugf("Registry isn't responding: trying another registry endpoint") continue } else if res.StatusCode == 404 { return nil, fmt.Errorf("Repository not found") From 09f1cbabb9ebc46776a832e1e695af7ef32a3de3 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 6 May 2013 11:06:44 -0700 Subject: [PATCH 36/36] Fixed imports --- graph.go | 1 + registry.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/graph.go b/graph.go index 853f8c736b..21d9d9407c 100644 --- a/graph.go +++ b/graph.go @@ -2,6 +2,7 @@ package docker import ( "fmt" + "io" "io/ioutil" "net/http" "os" diff --git a/registry.go b/registry.go index 72379dd3dd..1028d72725 100644 --- a/registry.go +++ b/registry.go @@ -9,7 +9,6 @@ import ( "io" "io/ioutil" "net/http" - "os" "path" "strings" )