diff --git a/auth/auth.go b/auth/auth.go index 2eb63b1d4a..5a5987ace8 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -3,7 +3,6 @@ package auth import ( "encoding/base64" "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -16,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const REGISTRY_SERVER = "https://registry.docker.io" +const INDEX_SERVER = "https://index.docker.io" type AuthConfig struct { Username string `json:"username"` @@ -76,6 +75,9 @@ func LoadConfig(rootPath string) (*AuthConfig, error) { return nil, err } arr := strings.Split(string(b), "\n") + if len(arr) < 2 { + return nil, fmt.Errorf("The Auth config file is empty") + } origAuth := strings.Split(arr[0], " = ") origEmail := strings.Split(arr[1], " = ") authConfig, err := DecodeAuth(origAuth[1]) @@ -89,9 +91,14 @@ func LoadConfig(rootPath string) (*AuthConfig, error) { // save the auth config func saveConfig(rootPath, authStr string, email string) error { + confFile := path.Join(rootPath, CONFIGFILE) + if len(email) == 0 { + os.Remove(confFile) + return nil + } lines := "auth = " + authStr + "\n" + "email = " + email + "\n" b := []byte(lines) - err := ioutil.WriteFile(path.Join(rootPath, CONFIGFILE), b, 0600) + err := ioutil.WriteFile(confFile, b, 0600) if err != nil { return err } @@ -101,40 +108,38 @@ func saveConfig(rootPath, authStr string, email string) error { // try to register/login to the registry server func Login(authConfig *AuthConfig) (string, error) { storeConfig := false + client := &http.Client{} reqStatusCode := 0 var status string - var errMsg string var reqBody []byte jsonBody, err := json.Marshal(authConfig) if err != nil { - errMsg = fmt.Sprintf("Config Error: %s", err) - return "", errors.New(errMsg) + return "", fmt.Errorf("Config Error: %s", err) } // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := http.Post(REGISTRY_SERVER+"/v1/users", "application/json; charset=utf-8", b) + req1, err := http.Post(INDEX_SERVER+"/v1/users/", "application/json; charset=utf-8", b) if err != nil { - errMsg = fmt.Sprintf("Server Error: %s", err) - return "", errors.New(errMsg) + return "", fmt.Errorf("Server Error: %s", err) } - reqStatusCode = req1.StatusCode defer req1.Body.Close() reqBody, err = ioutil.ReadAll(req1.Body) if err != nil { - errMsg = fmt.Sprintf("Server Error: [%#v] %s", reqStatusCode, err) - return "", errors.New(errMsg) + return "", fmt.Errorf("Server Error: [%#v] %s", reqStatusCode, err) } if reqStatusCode == 201 { - status = "Account Created\n" + status = "Account created. Please use the confirmation link we sent" + + " to your e-mail to activate it.\n" storeConfig = true + } else if reqStatusCode == 403 { + return "", fmt.Errorf("Login: Your account hasn't been activated. " + + "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { - // FIXME: This should be 'exists', not 'exist'. Need to change on the server first. - if string(reqBody) == "Username or email already exist" { - client := &http.Client{} - req, err := http.NewRequest("GET", REGISTRY_SERVER+"/v1/users", nil) + if string(reqBody) == "\"Username or email already exists\"" { + req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { @@ -148,17 +153,18 @@ func Login(authConfig *AuthConfig) (string, error) { if resp.StatusCode == 200 { status = "Login Succeeded\n" storeConfig = true + } else if resp.StatusCode == 401 { + saveConfig(authConfig.rootPath, "", "") + return "", fmt.Errorf("Wrong login/password, please try again") } else { - status = fmt.Sprintf("Login: %s", body) - return "", errors.New(status) + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, + resp.StatusCode, resp.Header) } } else { - status = fmt.Sprintf("Registration: %s", reqBody) - return "", errors.New(status) + return "", fmt.Errorf("Registration: %s", reqBody) } } else { - status = fmt.Sprintf("[%s] : %s", reqStatusCode, reqBody) - return "", errors.New(status) + return "", fmt.Errorf("Unexpected status code [%d] : %s", reqStatusCode, reqBody) } if storeConfig { authStr := EncodeAuth(authConfig) diff --git a/commands.go b/commands.go index 95a6753b95..48125992f6 100644 --- a/commands.go +++ b/commands.go @@ -521,6 +521,7 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout rcli.DockerConn, args . func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { cmd := rcli.Subcmd(stdout, "push", "NAME", "Push an image or a repository to the registry") + registry := cmd.String("registry", "", "Registry host to push the image to") if err := cmd.Parse(args); err != nil { return nil } @@ -531,8 +532,8 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ... return nil } - // If the login failed, abort - if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { + // If the login failed AND we're using the index, abort + if *registry == "" && (srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "") { if err := srv.CmdLogin(stdin, stdout, args...); err != nil { return err } @@ -555,9 +556,6 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ... Debugf("Pushing [%s] to [%s]\n", local, remote) // Try to get the image - // FIXME: Handle lookup - // FIXME: Also push the tags in case of ./docker push myrepo:mytag - // img, err := srv.runtime.LookupImage(cmd.Arg(0)) img, err := srv.runtime.graph.Get(local) if err != nil { Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) @@ -571,7 +569,7 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ... return err } - err = srv.runtime.graph.PushImage(stdout, img, srv.runtime.authConfig) + err = srv.runtime.graph.PushImage(stdout, img, *registry, nil) if err != nil { return err } @@ -580,6 +578,8 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ... func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "NAME", "Pull an image or a repository from the registry") + tag := cmd.String("t", "", "Download tagged image in repository") + registry := cmd.String("registry", "", "Registry to download from. Necessary if image is pulled by ID") if err := cmd.Parse(args); err != nil { return nil } @@ -589,15 +589,20 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } + if strings.Contains(remote, ":") { + remoteParts := strings.Split(remote, ":") + tag = &remoteParts[1] + remote = remoteParts[0] + } + // FIXME: CmdPull should be a wrapper around Runtime.Pull() - if srv.runtime.graph.LookupRemoteImage(remote, srv.runtime.authConfig) { - if err := srv.runtime.graph.PullImage(stdout, remote, srv.runtime.authConfig); err != nil { + if *registry != "" { + if err := srv.runtime.graph.PullImage(stdout, remote, *registry, nil); err != nil { return err } return nil } - // FIXME: Allow pull repo:tag - if err := srv.runtime.graph.PullRepository(stdout, remote, "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { + if err := srv.runtime.graph.PullRepository(stdout, remote, *tag, srv.runtime.repositories, srv.runtime.authConfig); err != nil { return err } return nil diff --git a/container.go b/container.go index dc57a31135..311e475ac1 100644 --- a/container.go +++ b/container.go @@ -736,6 +736,14 @@ func (container *Container) ExportRw() (Archive, error) { return Tar(container.rwPath(), Uncompressed) } +func (container *Container) RwChecksum() (string, error) { + rwData, err := Tar(container.rwPath(), Xz) + if err != nil { + return "", err + } + return HashData(rwData) +} + func (container *Container) Export() (Archive, error) { if err := container.EnsureMounted(); err != nil { return nil, err diff --git a/graph.go b/graph.go index 3823868c83..21d9d9407c 100644 --- a/graph.go +++ b/graph.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "os" "path" "path/filepath" @@ -13,8 +14,9 @@ import ( // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *TruncIndex + Root string + idIndex *TruncIndex + httpClient *http.Client } // NewGraph instantiates a new graph at the given root path in the filesystem. @@ -97,15 +99,11 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Parent = container.Image img.Container = container.Id img.ContainerConfig = *container.Config - if config == nil { - if parentImage, err := graph.Get(container.Image); err == nil && parentImage != nil { - img.Config = parentImage.Config - } - } } if err := graph.Register(layerData, img); err != nil { return nil, err } + img.Checksum() return img, nil } diff --git a/image.go b/image.go index 09c0f8dcf6..bf86e2e7f7 100644 --- a/image.go +++ b/image.go @@ -2,6 +2,7 @@ package docker import ( "crypto/rand" + "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -51,6 +52,7 @@ func LoadImage(root string) (*Image, error) { } else if !stat.IsDir() { return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root)) } + return &img, nil } @@ -257,3 +259,62 @@ func (img *Image) layer() (string, error) { } return layerPath(root), nil } + +func (img *Image) Checksum() (string, error) { + root, err := img.root() + if err != nil { + return "", err + } + + checksumDictPth := path.Join(root, "..", "..", "checksums") + checksums := new(map[string]string) + + if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { + if err := json.Unmarshal(checksumDict, checksums); err != nil { + return "", err + } + if checksum, ok := (*checksums)[img.Id]; ok { + return checksum, nil + } + } + + layer, err := img.layer() + if err != nil { + return "", err + } + jsonData, err := ioutil.ReadFile(jsonPath(root)) + if err != nil { + return "", err + } + + layerData, err := Tar(layer, Xz) + if err != nil { + return "", err + } + + h := sha256.New() + if _, err := h.Write(jsonData); err != nil { + return "", err + } + if _, err := h.Write([]byte("\n")); err != nil { + return "", err + } + if _, err := io.Copy(h, layerData); err != nil { + return "", err + } + + hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) + if *checksums == nil { + *checksums = map[string]string{} + } + (*checksums)[img.Id] = hash + checksumJson, err := json.Marshal(checksums) + if err != nil { + return hash, err + } + + if err := ioutil.WriteFile(checksumDictPth, checksumJson, 0600); err != nil { + return hash, err + } + return hash, nil +} diff --git a/registry.go b/registry.go index 74b166906f..1028d72725 100644 --- a/registry.go +++ b/registry.go @@ -1,20 +1,20 @@ package docker import ( + "bytes" "encoding/json" "fmt" "github.com/dotcloud/docker/auth" + "github.com/shin-/cookiejar" "io" "io/ioutil" "net/http" - "os" "path" "strings" ) //FIXME: Set the endpoint in a conf file or via commandline -//const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1" -const REGISTRY_ENDPOINT = auth.REGISTRY_SERVER + "/v1" +const INDEX_ENDPOINT = auth.INDEX_SERVER + "/v1" // Build an Image object from raw json data func NewImgJson(src []byte) (*Image, error) { @@ -28,34 +28,23 @@ func NewImgJson(src []byte) (*Image, error) { return ret, nil } -// Build an Image object list from a raw json data -// FIXME: Do this in "stream" mode -func NewMultipleImgJson(src []byte) ([]*Image, error) { - ret := []*Image{} - - dec := json.NewDecoder(strings.NewReader(string(src))) - for { - m := &Image{} - if err := dec.Decode(m); err == io.EOF { - break - } else if err != nil { - return nil, err - } - ret = append(ret, m) +func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { + for _, cookie := range c.Jar.Cookies(req.URL) { + req.AddCookie(cookie) } - return ret, nil + return c.Do(req) } // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) ([]*Image, error) { - client := &http.Client{} +func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]string, error) { + client := graph.getHttpClient() - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/history", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -70,41 +59,83 @@ func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) return nil, fmt.Errorf("Error while reading the http response: %s\n", err) } - history, err := NewMultipleImgJson(jsonString) - if err != nil { - return nil, fmt.Errorf("Error while parsing the json: %s\n", err) + Debugf("Ancestry: %s", jsonString) + history := new([]string) + if err := json.Unmarshal(jsonString, history); err != nil { + return nil, err } - return history, nil + return *history, nil +} + +func (graph *Graph) getHttpClient() *http.Client { + if graph.httpClient == nil { + graph.httpClient = new(http.Client) + graph.httpClient.Jar = cookiejar.NewCookieJar() + } + return graph.httpClient } // Check if an image exists in the Registry -func (graph *Graph) LookupRemoteImage(imgId string, authConfig *auth.AuthConfig) bool { +func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return false } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := rt.RoundTrip(req) - if err != nil || res.StatusCode != 307 { - return false + return err == nil && res.StatusCode == 307 +} + +func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { + u := INDEX_ENDPOINT + "/repositories/" + repository + "/images" + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err } - return res.StatusCode == 307 + if authConfig != nil && len(authConfig.Username) > 0 { + req.SetBasicAuth(authConfig.Username, authConfig.Password) + } + res, err := graph.getHttpClient().Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // Repository doesn't exist yet + if res.StatusCode == 404 { + return nil, nil + } + + jsonData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + imageList := []map[string]string{} + + err = json.Unmarshal(jsonData, &imageList) + if err != nil { + Debugf("Body: %s (%s)\n", res.Body, u) + return nil, err + } + + return imageList, nil } // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) -func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *auth.AuthConfig) (*Image, Archive, error) { - client := &http.Client{} +func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, token []string) (*Image, Archive, error) { + client := graph.getHttpClient() fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) // Get the Json - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -127,11 +158,11 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *a // Get the layer fmt.Fprintf(stdout, "Pulling %s fs layer\r\n", imgId) - req, err = http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/layer", nil) + req, err = http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -139,16 +170,87 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *a return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil } -func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.AuthConfig) error { - history, err := graph.getRemoteHistory(imgId, authConfig) +func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { + client := graph.getHttpClient() + if strings.Count(repository, "/") == 0 { + // This will be removed once the Registry supports auto-resolution on + // the "library" namespace + repository = "library/" + repository + } + for _, host := range registries { + endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := client.Do(req) + defer res.Body.Close() + Debugf("Got status code %d from %s", res.StatusCode, endpoint) + if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { + continue + } else if res.StatusCode == 404 { + return nil, fmt.Errorf("Repository not found") + } + + result := new(map[string]string) + + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + if err = json.Unmarshal(rawJson, result); err != nil { + return nil, err + } + + return *result, nil + + } + return nil, fmt.Errorf("Could not reach any registry endpoint") +} + +func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry string, token []string) (string, error) { + client := graph.getHttpClient() + registryEndpoint := "https://" + registry + "/v1" + repositoryTarget := registryEndpoint + "/repositories/" + remote + "/tags/" + tag + + req, err := http.NewRequest("GET", repositoryTarget, nil) + if err != nil { + return "", err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("Error while retrieving repository info: %v", err) + } + defer res.Body.Close() + if res.StatusCode == 403 { + return "", fmt.Errorf("You aren't authorized to access this resource") + } else if res.StatusCode != 200 { + return "", fmt.Errorf("HTTP code: %d", res.StatusCode) + } + + var imgId string + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + if err = json.Unmarshal(rawJson, &imgId); err != nil { + return "", err + } + return imgId, nil +} + +func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token []string) error { + history, err := graph.getRemoteHistory(imgId, registry, token) if err != nil { return err } // FIXME: Try to stream the images? - // FIXME: Lunch the getRemoteImage() in goroutines - for _, j := range history { - if !graph.Exists(j.Id) { - img, layer, err := graph.getRemoteImage(stdout, j.Id, authConfig) + // FIXME: Launch the getRemoteImage() in goroutines + for _, id := range history { + if !graph.Exists(id) { + img, layer, err := graph.getRemoteImage(stdout, id, registry, token) if err != nil { // FIXME: Keep goging in case of error? return err @@ -161,165 +263,195 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.A return nil } -// FIXME: Handle the askedTag parameter func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { - client := &http.Client{} + client := graph.getHttpClient() - fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - - var repositoryTarget string - // If we are asking for 'root' repository, lookup on the Library's registry - if strings.Index(remote, "/") == -1 { - repositoryTarget = REGISTRY_ENDPOINT + "/library/" + remote - } else { - repositoryTarget = REGISTRY_ENDPOINT + "/users/" + remote - } + fmt.Fprintf(stdout, "Pulling repository %s from %s\r\n", remote, INDEX_ENDPOINT) + repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/images" req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { return err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + if authConfig != nil && len(authConfig.Username) > 0 { + req.SetBasicAuth(authConfig.Username, authConfig.Password) + } + req.Header.Set("X-Docker-Token", "true") + res, err := client.Do(req) if err != nil { return err } defer res.Body.Close() + if res.StatusCode == 401 { + return fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + } + // TODO: Right now we're ignoring checksums in the response body. + // In the future, we need to use them to check image validity. if res.StatusCode != 200 { return fmt.Errorf("HTTP code: %d", res.StatusCode) } - rawJson, err := ioutil.ReadAll(res.Body) + + var token, endpoints []string + if res.Header.Get("X-Docker-Token") != "" { + token = res.Header["X-Docker-Token"] + } + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return fmt.Errorf("Index response didn't contain any endpoints") + } + + var tagsList map[string]string + if askedTag == "" { + tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) + if err != nil { + return err + } + } else { + tagsList = map[string]string{askedTag: ""} + } + + for askedTag, imgId := range tagsList { + fmt.Fprintf(stdout, "Resolving tag \"%s:%s\" from %s\n", remote, askedTag, endpoints) + success := false + for _, registry := range endpoints { + if imgId == "" { + imgId, err = graph.getImageForTag(stdout, askedTag, remote, registry, token) + if err != nil { + fmt.Fprintf(stdout, "Error while retrieving image for tag: %v (%v) ; "+ + "checking next endpoint", askedTag, err) + continue + } + } + + if err := graph.PullImage(stdout, imgId, "https://"+registry+"/v1", token); err != nil { + return err + } + + if err = repositories.Set(remote, askedTag, imgId, true); err != nil { + return err + } + success = true + } + + if !success { + return fmt.Errorf("Could not find repository on any of the indexed registries.") + } + } + + if err = repositories.Save(); err != nil { + return err + } + + return nil +} + +func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, token []string) error { + if parent, err := img.GetParent(); err != nil { + return err + } else if parent != nil { + if err := pushImageRec(graph, stdout, parent, registry, token); err != nil { + return err + } + } + client := graph.getHttpClient() + jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) + if err != nil { + return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) + } + + fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) + + // FIXME: try json with UTF8 + jsonData := strings.NewReader(string(jsonRaw)) + req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) if err != nil { return err } - t := map[string]string{} - if err = json.Unmarshal(rawJson, &t); err != nil { + req.Header.Add("Content-type", "application/json") + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + + checksum, err := img.Checksum() + if err != nil { + return fmt.Errorf("Error while retrieving checksum for %s: %v", img.Id, err) + } + req.Header.Set("X-Docker-Checksum", checksum) + res, err := doWithCookies(client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ + " trying to parse response body: %v", res.StatusCode, err) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + fmt.Fprintf(stdout, "Image %v already uploaded ; skipping\n", img.Id) + return nil + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + + fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) + + layerData, err := graph.TempLayerArchive(img.Id, Xz, stdout) + if err != nil { + return fmt.Errorf("Failed to generate layer archive: %s", err) + } + + req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", + ProgressReader(layerData, -1, stdout, "")) + if err != nil { return err } - for tag, rev := range t { - fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) - if err = graph.PullImage(stdout, rev, authConfig); err != nil { - return err - } - if err = repositories.Set(remote, tag, rev, true); err != nil { - return err - } + + req3.ContentLength = -1 + req3.TransferEncoding = []string{"chunked"} + req3.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + res3, err := doWithCookies(client, req3) + if err != nil { + return fmt.Errorf("Failed to upload layer: %s", err) } - if err = repositories.Save(); err != nil { - return err + res3.Body.Close() + if res3.StatusCode != 200 { + return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) } return nil } // Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth.AuthConfig) error { - client := &http.Client{} - - // FIXME: Factorize the code - // FIXME: Do the puts in goroutines - if err := imgOrig.WalkHistory(func(img *Image) error { - - jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) - if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) - } - - fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) - - // FIXME: try json with UTF8 - jsonData := strings.NewReader(string(jsonRaw)) - req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(authConfig.Username, authConfig.Password) - res, err := client.Do(req) - if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - switch res.StatusCode { - case 204: - // Case where the image is already on the Registry - // FIXME: Do not be silent? - return nil - default: - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - errBody = []byte(err.Error()) - } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) - } - } - - fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil) - req2.SetBasicAuth(authConfig.Username, authConfig.Password) - res2, err := client.Do(req2) - if err != nil { - return fmt.Errorf("Registry returned error: %s", err) - } - res2.Body.Close() - if res2.StatusCode != 307 { - return fmt.Errorf("Registry returned unexpected HTTP status code %d, expected 307", res2.StatusCode) - } - url, err := res2.Location() - if err != nil || url == nil { - return fmt.Errorf("Failed to retrieve layer upload location: %s", err) - } - - // FIXME: stream the archive directly to the registry instead of buffering it on disk. This requires either: - // a) Implementing S3's proprietary streaming logic, or - // b) Stream directly to the registry instead of S3. - // I prefer option b. because it doesn't lock us into a proprietary cloud service. - tmpLayer, err := graph.TempLayerArchive(img.Id, Xz, stdout) - if err != nil { - return err - } - defer os.Remove(tmpLayer.Name()) - req3, err := http.NewRequest("PUT", url.String(), ProgressReader(tmpLayer, int(tmpLayer.Size), stdout, "Uploading %v/%v (%v)")) - if err != nil { - return err - } - req3.ContentLength = int64(tmpLayer.Size) - - req3.TransferEncoding = []string{"none"} - res3, err := client.Do(req3) - if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) - } - res3.Body.Close() - if res3.StatusCode != 200 { - return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) - } - return nil - }); err != nil { - return err - } - return nil +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error { + registry = "https://" + registry + "/v1" + return pushImageRec(graph, stdout, imgOrig, registry, token) } // push a tag on the registry. // Remote has the format '/ -func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthConfig) error { - - // Keep this for backward compatibility - if tag == "" { - tag = "lastest" - } - +func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" + registry = "https://" + registry + "/v1" - Debugf("Pushing tags for rev [%s] on {%s}\n", revision, REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag) + Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) - client := &http.Client{} - req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag, strings.NewReader(revision)) + client := graph.getHttpClient() + req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + if err != nil { + return err + } req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(authConfig.Username, authConfig.Password) - res, err := client.Do(req) + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + req.ContentLength = int64(len(revision)) + res, err := doWithCookies(client, req) if err != nil { return err } @@ -327,62 +459,25 @@ func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthC if res.StatusCode != 200 && res.StatusCode != 201 { return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) } - Debugf("Result of push tag: %d\n", res.StatusCode) - switch res.StatusCode { - default: - return fmt.Errorf("Error %d\n", res.StatusCode) - case 200: - case 201: - } return nil } -func (graph *Graph) LookupRemoteRepository(remote string, authConfig *auth.AuthConfig) bool { - rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - - var repositoryTarget string - // If we are asking for 'root' repository, lookup on the Library's registry - if strings.Index(remote, "/") == -1 { - repositoryTarget = REGISTRY_ENDPOINT + "/library/" + remote + "/lookup" - } else { - repositoryTarget = REGISTRY_ENDPOINT + "/users/" + remote + "/lookup" - } - Debugf("Checking for permissions on: %s", repositoryTarget) - req, err := http.NewRequest("PUT", repositoryTarget, strings.NewReader("\"\"")) - if err != nil { - Debugf("%s\n", err) - return false - } - req.SetBasicAuth(authConfig.Username, authConfig.Password) - req.Header.Add("Content-type", "application/json") - res, err := rt.RoundTrip(req) - if err != nil || res.StatusCode != 404 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - errBody = []byte(err.Error()) - } - Debugf("Lookup status code: %d (body: %s)", res.StatusCode, errBody) - return false - } - return true -} - // FIXME: this should really be PushTag -func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, token []string) error { // Check if the local impage exists img, err := graph.Get(imgId) if err != nil { fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId) return nil } - fmt.Fprintf(stdout, "Pushing tag %s:%s\r\n", remote, tag) + fmt.Fprintf(stdout, "Pushing image %s:%s\r\n", remote, tag) // Push the image - if err = graph.PushImage(stdout, img, authConfig); err != nil { + if err = graph.PushImage(stdout, img, registry, token); err != nil { return err } fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) // And then the tag - if err = graph.pushTag(remote, imgId, tag, authConfig); err != nil { + if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil { return err } return nil @@ -391,18 +486,155 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, a // Push a repository to the registry. // Remote has the format '/ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { - // Check if the remote repository exists/if we have the permission - if !graph.LookupRemoteRepository(remote, authConfig) { - return fmt.Errorf("Permission denied on repository %s\n", remote) + client := graph.getHttpClient() + + checksums, err := graph.Checksums(stdout, localRepo) + if err != nil { + return err } - fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) - // For each image within the repo, push them - for tag, imgId := range localRepo { - if err := graph.pushPrimitive(stdout, remote, tag, imgId, authConfig); err != nil { - // FIXME: Continue on error? - return err + imgList := make([]map[string]string, len(checksums)) + checksums2 := make([]map[string]string, len(checksums)) + + uploadedImages, err := graph.getImagesInRepository(remote, authConfig) + if err != nil { + return fmt.Errorf("Error occured while fetching the list: %s", err) + } + + // Filter list to only send images/checksums not already uploaded + i := 0 + for _, obj := range checksums { + found := false + for _, uploadedImg := range uploadedImages { + if obj["id"] == uploadedImg["id"] && uploadedImg["checksum"] != "" { + found = true + break + } + } + if !found { + imgList[i] = map[string]string{"id": obj["id"]} + checksums2[i] = obj + i += 1 } } + checksums = checksums2[:i] + imgList = imgList[:i] + + imgListJson, err := json.Marshal(imgList) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/", bytes.NewReader(imgListJson)) + if err != nil { + return err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.ContentLength = int64(len(imgListJson)) + req.Header.Set("X-Docker-Token", "true") + res, err := client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + for res.StatusCode >= 300 && res.StatusCode < 400 { + Debugf("Redirected to %s\n", res.Header.Get("Location")) + req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) + if err != nil { + return err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.ContentLength = int64(len(imgListJson)) + req.Header.Set("X-Docker-Token", "true") + res, err = client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + } + + if res.StatusCode != 200 && res.StatusCode != 201 { + return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote) + } + + var token, endpoints []string + if res.Header.Get("X-Docker-Token") != "" { + token = res.Header["X-Docker-Token"] + Debugf("Auth token: %v", token) + } else { + return fmt.Errorf("Index response didn't contain an access token") + } + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return fmt.Errorf("Index response didn't contain any endpoints") + } + + for _, registry := range endpoints { + fmt.Fprintf(stdout, "Pushing repository %s to %s (%d tags)\r\n", remote, registry, + len(localRepo)) + // For each image within the repo, push them + for tag, imgId := range localRepo { + if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, token); err != nil { + // FIXME: Continue on error? + return err + } + } + } + checksumsJson, err := json.Marshal(checksums) + if err != nil { + return err + } + + req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(checksumsJson)) + if err != nil { + return err + } + req2.SetBasicAuth(authConfig.Username, authConfig.Password) + req2.Header["X-Docker-Endpoints"] = endpoints + req2.ContentLength = int64(len(checksumsJson)) + res2, err := client.Do(req2) + if err != nil { + return err + } + res2.Body.Close() + if res2.StatusCode != 204 { + return fmt.Errorf("Error: Status %d trying to push checksums %s", res.StatusCode, remote) + } + return nil } + +func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]string, error) { + var result []map[string]string + checksums := map[string]string{} + for _, id := range repo { + img, err := graph.Get(id) + if err != nil { + return nil, err + } + err = img.WalkHistory(func(image *Image) error { + fmt.Fprintf(output, "Computing checksum for image %s\n", image.Id) + if _, exists := checksums[image.Id]; !exists { + checksums[image.Id], err = image.Checksum() + if err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, err + } + } + i := 0 + result = make([]map[string]string, len(checksums)) + for id, sum := range checksums { + result[i] = map[string]string{ + "id": id, + "checksum": sum, + } + i++ + } + return result, nil +} diff --git a/utils.go b/utils.go index 4bdbd1253b..229b938830 100644 --- a/utils.go +++ b/utils.go @@ -2,6 +2,8 @@ package docker import ( "bytes" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "github.com/dotcloud/docker/rcli" @@ -395,6 +397,15 @@ func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) return written, err } + +func HashData(src io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + type KernelVersionInfo struct { Kernel int Major int