diff --git a/graph.go b/graph.go index 9a8fd1b3cb..84b8c9a1e9 100644 --- a/graph.go +++ b/graph.go @@ -9,14 +9,18 @@ import ( "path" "path/filepath" "strings" + "sync" "time" ) // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *TruncIndex - httpClient *http.Client + Root string + idIndex *TruncIndex + httpClient *http.Client + checksumLock map[string]*sync.Mutex + lockSumFile *sync.Mutex + lockSumMap *sync.Mutex } // NewGraph instantiates a new graph at the given root path in the filesystem. @@ -27,12 +31,15 @@ func NewGraph(root string) (*Graph, error) { return nil, err } // Create the root directory if it doesn't exists - if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { return nil, err } graph := &Graph{ - Root: abspath, - idIndex: NewTruncIndex(), + Root: abspath, + idIndex: NewTruncIndex(), + checksumLock: make(map[string]*sync.Mutex), + lockSumFile: &sync.Mutex{}, + lockSumMap: &sync.Mutex{}, } if err := graph.restore(); err != nil { return nil, err @@ -82,6 +89,11 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) } img.graph = graph + graph.lockSumMap.Lock() + defer graph.lockSumMap.Unlock() + if _, exists := graph.checksumLock[img.Id]; !exists { + graph.checksumLock[img.Id] = &sync.Mutex{} + } return img, nil } @@ -103,7 +115,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut if err := graph.Register(layerData, img); err != nil { return nil, err } - img.Checksum() + go img.Checksum() return img, nil } @@ -131,6 +143,7 @@ func (graph *Graph) Register(layerData Archive, img *Image) error { } img.graph = graph graph.idIndex.Add(img.Id) + graph.checksumLock[img.Id] = &sync.Mutex{} return nil } diff --git a/image.go b/image.go index bf86e2e7f7..d208ed6ce8 100644 --- a/image.go +++ b/image.go @@ -35,8 +35,9 @@ func LoadImage(root string) (*Image, error) { if err != nil { return nil, err } - var img Image - if err := json.Unmarshal(jsonData, &img); err != nil { + img := &Image{} + + if err := json.Unmarshal(jsonData, img); err != nil { return nil, err } if err := ValidateId(img.Id); err != nil { @@ -52,8 +53,7 @@ func LoadImage(root string) (*Image, error) { } else if !stat.IsDir() { return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root)) } - - return &img, nil + return img, nil } func StoreImage(img *Image, layerData Archive, root string) error { @@ -261,19 +261,22 @@ func (img *Image) layer() (string, error) { } func (img *Image) Checksum() (string, error) { + img.graph.checksumLock[img.Id].Lock() + defer img.graph.checksumLock[img.Id].Unlock() + root, err := img.root() if err != nil { return "", err } checksumDictPth := path.Join(root, "..", "..", "checksums") - checksums := new(map[string]string) + checksums := make(map[string]string) if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { - if err := json.Unmarshal(checksumDict, checksums); err != nil { + if err := json.Unmarshal(checksumDict, &checksums); err != nil { return "", err } - if checksum, ok := (*checksums)[img.Id]; ok { + if checksum, ok := checksums[img.Id]; ok { return checksum, nil } } @@ -299,20 +302,26 @@ func (img *Image) Checksum() (string, error) { if _, err := h.Write([]byte("\n")); err != nil { return "", err } + if _, err := io.Copy(h, layerData); err != nil { return "", err } hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) - if *checksums == nil { - *checksums = map[string]string{} + checksums[img.Id] = hash + + // Reload the json file to make sure not to overwrite faster sums + img.graph.lockSumFile.Lock() + defer img.graph.lockSumFile.Unlock() + if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { + if err := json.Unmarshal(checksumDict, &checksums); err != nil { + return "", err + } } - (*checksums)[img.Id] = hash checksumJson, err := json.Marshal(checksums) if err != nil { return hash, err } - if err := ioutil.WriteFile(checksumDictPth, checksumJson, 0600); err != nil { return hash, err } diff --git a/registry.go b/registry.go index cc8ee68153..bf6556f6d5 100644 --- a/registry.go +++ b/registry.go @@ -194,18 +194,16 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit return nil, fmt.Errorf("Repository not found") } - result := new(map[string]string) + result := make(map[string]string) rawJson, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err } - if err = json.Unmarshal(rawJson, result); err != nil { + if err = json.Unmarshal(rawJson, &result); err != nil { return nil, err } - - return *result, nil - + return result, nil } return nil, fmt.Errorf("Could not reach any registry endpoint") } @@ -308,6 +306,50 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return fmt.Errorf("Index response didn't contain any endpoints") } + checksumsJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + + // Reload the json file to make sure not to overwrite faster sums + err = func() error { + localChecksums := make(map[string]string) + remoteChecksums := []struct { + Id string `json: "id"` + Checksum string `json: "checksum"` + }{} + checksumDictPth := path.Join(graph.Root, "..", "checksums") + + if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil { + return err + } + + graph.lockSumFile.Lock() + defer graph.lockSumFile.Unlock() + + if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { + if err := json.Unmarshal(checksumDict, &localChecksums); err != nil { + return err + } + } + + for _, elem := range remoteChecksums { + localChecksums[elem.Id] = elem.Checksum + } + + checksumsJson, err = json.Marshal(localChecksums) + if err != nil { + return err + } + if err := ioutil.WriteFile(checksumDictPth, checksumsJson, 0600); err != nil { + return err + } + return nil + }() + if err != nil { + return err + } + var tagsList map[string]string if askedTag == "" { tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) @@ -427,9 +469,15 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) } - res3.Body.Close() + defer res3.Body.Close() + if res3.StatusCode != 200 { - return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) + errBody, err := ioutil.ReadAll(res3.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ + " trying to parse response body: %v", res.StatusCode, err) + } + return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody) } return nil } @@ -612,8 +660,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re } func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]string, error) { - var result []map[string]string - checksums := map[string]string{} + checksums := make(map[string]string) for _, id := range repo { img, err := graph.Get(id) if err != nil { @@ -634,7 +681,7 @@ func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]s } } i := 0 - result = make([]map[string]string, len(checksums)) + result := make([]map[string]string, len(checksums)) for id, sum := range checksums { result[i] = map[string]string{ "id": id,