Merge pull request #557 from dotcloud/improve_checksum

* Registry: Improve checksum

graph.go

@@ -9,14 +9,18 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 )
 
 // A Graph is a store for versioned filesystem images and the relationship between them.
 type Graph struct {
-	Root       string
-	idIndex    *TruncIndex
-	httpClient *http.Client
+	Root         string
+	idIndex      *TruncIndex
+	httpClient   *http.Client
+	checksumLock map[string]*sync.Mutex
+	lockSumFile  *sync.Mutex
+	lockSumMap   *sync.Mutex
 }
 
 // NewGraph instantiates a new graph at the given root path in the filesystem.
@@ -27,12 +31,15 @@ func NewGraph(root string) (*Graph, error) {
 		return nil, err
 	}
 	// Create the root directory if it doesn't exists
-	if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) {
+	if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) {
 		return nil, err
 	}
 	graph := &Graph{
-		Root:    abspath,
-		idIndex: NewTruncIndex(),
+		Root:         abspath,
+		idIndex:      NewTruncIndex(),
+		checksumLock: make(map[string]*sync.Mutex),
+		lockSumFile:  &sync.Mutex{},
+		lockSumMap:   &sync.Mutex{},
 	}
 	if err := graph.restore(); err != nil {
 		return nil, err
@@ -82,6 +89,11 @@ func (graph *Graph) Get(name string) (*Image, error) {
 		return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id)
 	}
 	img.graph = graph
+	graph.lockSumMap.Lock()
+	defer graph.lockSumMap.Unlock()
+	if _, exists := graph.checksumLock[img.Id]; !exists {
+		graph.checksumLock[img.Id] = &sync.Mutex{}
+	}
 	return img, nil
 }
 
@@ -103,7 +115,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut
 	if err := graph.Register(layerData, img); err != nil {
 		return nil, err
 	}
-	img.Checksum()
+	go img.Checksum()
 	return img, nil
 }
 
@@ -131,6 +143,7 @@ func (graph *Graph) Register(layerData Archive, img *Image) error {
 	}
 	img.graph = graph
 	graph.idIndex.Add(img.Id)
+	graph.checksumLock[img.Id] = &sync.Mutex{}
 	return nil
 }
 

image.go

@@ -35,8 +35,9 @@ func LoadImage(root string) (*Image, error) {
 	if err != nil {
 		return nil, err
 	}
-	var img Image
-	if err := json.Unmarshal(jsonData, &img); err != nil {
+	img := &Image{}
+
+	if err := json.Unmarshal(jsonData, img); err != nil {
 		return nil, err
 	}
 	if err := ValidateId(img.Id); err != nil {
@@ -52,8 +53,7 @@ func LoadImage(root string) (*Image, error) {
 	} else if !stat.IsDir() {
 		return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root))
 	}
-
-	return &img, nil
+	return img, nil
 }
 
 func StoreImage(img *Image, layerData Archive, root string) error {
@@ -261,19 +261,22 @@ func (img *Image) layer() (string, error) {
 }
 
 func (img *Image) Checksum() (string, error) {
+	img.graph.checksumLock[img.Id].Lock()
+	defer img.graph.checksumLock[img.Id].Unlock()
+
 	root, err := img.root()
 	if err != nil {
 		return "", err
 	}
 
 	checksumDictPth := path.Join(root, "..", "..", "checksums")
-	checksums := new(map[string]string)
+	checksums := make(map[string]string)
 
 	if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil {
-		if err := json.Unmarshal(checksumDict, checksums); err != nil {
+		if err := json.Unmarshal(checksumDict, &checksums); err != nil {
 			return "", err
 		}
-		if checksum, ok := (*checksums)[img.Id]; ok {
+		if checksum, ok := checksums[img.Id]; ok {
 			return checksum, nil
 		}
 	}
@@ -299,20 +302,26 @@ func (img *Image) Checksum() (string, error) {
 	if _, err := h.Write([]byte("\n")); err != nil {
 		return "", err
 	}
+
 	if _, err := io.Copy(h, layerData); err != nil {
 		return "", err
 	}
 
 	hash := "sha256:" + hex.EncodeToString(h.Sum(nil))
-	if *checksums == nil {
-		*checksums = map[string]string{}
+	checksums[img.Id] = hash
+
+	// Reload the json file to make sure not to overwrite faster sums
+	img.graph.lockSumFile.Lock()
+	defer img.graph.lockSumFile.Unlock()
+	if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil {
+		if err := json.Unmarshal(checksumDict, &checksums); err != nil {
+			return "", err
+		}
 	}
-	(*checksums)[img.Id] = hash
 	checksumJson, err := json.Marshal(checksums)
 	if err != nil {
 		return hash, err
 	}
-
 	if err := ioutil.WriteFile(checksumDictPth, checksumJson, 0600); err != nil {
 		return hash, err
 	}

registry.go

@@ -194,18 +194,16 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit
 			return nil, fmt.Errorf("Repository not found")
 		}
 
-		result := new(map[string]string)
+		result := make(map[string]string)
 
 		rawJson, err := ioutil.ReadAll(res.Body)
 		if err != nil {
 			return nil, err
 		}
-		if err = json.Unmarshal(rawJson, result); err != nil {
+		if err = json.Unmarshal(rawJson, &result); err != nil {
 			return nil, err
 		}
-
-		return *result, nil
-
+		return result, nil
 	}
 	return nil, fmt.Errorf("Could not reach any registry endpoint")
 }
@@ -308,6 +306,50 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re
 		return fmt.Errorf("Index response didn't contain any endpoints")
 	}
 
+	checksumsJson, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+
+	// Reload the json file to make sure not to overwrite faster sums
+	err = func() error {
+		localChecksums := make(map[string]string)
+		remoteChecksums := []struct {
+			Id       string `json: "id"`
+			Checksum string `json: "checksum"`
+		}{}
+		checksumDictPth := path.Join(graph.Root, "..", "checksums")
+
+		if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil {
+			return err
+		}
+
+		graph.lockSumFile.Lock()
+		defer graph.lockSumFile.Unlock()
+
+		if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil {
+			if err := json.Unmarshal(checksumDict, &localChecksums); err != nil {
+				return err
+			}
+		}
+
+		for _, elem := range remoteChecksums {
+			localChecksums[elem.Id] = elem.Checksum
+		}
+
+		checksumsJson, err = json.Marshal(localChecksums)
+		if err != nil {
+			return err
+		}
+		if err := ioutil.WriteFile(checksumDictPth, checksumsJson, 0600); err != nil {
+			return err
+		}
+		return nil
+	}()
+	if err != nil {
+		return err
+	}
+
 	var tagsList map[string]string
 	if askedTag == "" {
 		tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token)
@@ -427,9 +469,15 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t
 	if err != nil {
 		return fmt.Errorf("Failed to upload layer: %s", err)
 	}
-	res3.Body.Close()
+	defer res3.Body.Close()
+
 	if res3.StatusCode != 200 {
-		return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode)
+		errBody, err := ioutil.ReadAll(res3.Body)
+		if err != nil {
+			return fmt.Errorf("HTTP code %d while uploading metadata and error when"+
+				" trying to parse response body: %v", res.StatusCode, err)
+		}
+		return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody)
 	}
 	return nil
 }
@@ -612,8 +660,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
 }
 
 func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]string, error) {
-	var result []map[string]string
-	checksums := map[string]string{}
+	checksums := make(map[string]string)
 	for _, id := range repo {
 		img, err := graph.Get(id)
 		if err != nil {
@@ -634,7 +681,7 @@ func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]s
 		}
 	}
 	i := 0
-	result = make([]map[string]string, len(checksums))
+	result := make([]map[string]string, len(checksums))
 	for id, sum := range checksums {
 		result[i] = map[string]string{
 			"id":       id,