From 2b58b677a54950bc78c13760ef79fe6284154847 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 5 Jun 2015 15:31:10 -0700 Subject: [PATCH 1/3] Separate graph from image Move graph related functions in image to graph package. Consolidating graph functionality is the first step in refactoring graph into an image store model. Subsequent refactors will involve breaking up graph into multiple types with a strongly defined interface. Signed-off-by: Derek McGowan (github: dmcgowan) --- daemon/create.go | 2 +- daemon/image_delete.go | 2 +- graph/graph.go | 170 +++++++++++++++++++++++++----- graph/graph_test.go | 8 +- graph/history.go | 91 +++++++++++++++- graph/list.go | 4 +- graph/manifest_test.go | 18 ++-- graph/push.go | 16 ++- graph/service.go | 6 +- image/graph.go | 11 -- image/image.go | 229 ----------------------------------------- 11 files changed, 259 insertions(+), 298 deletions(-) delete mode 100644 image/graph.go diff --git a/daemon/create.go b/daemon/create.go index 16b6b48b92..ba0a2bf0c4 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -59,7 +59,7 @@ func (daemon *Daemon) Create(config *runconfig.Config, hostConfig *runconfig.Hos if err != nil { return nil, nil, err } - if err = img.CheckDepth(); err != nil { + if err = daemon.graph.CheckDepth(img); err != nil { return nil, nil, err } imgID = img.ID diff --git a/daemon/image_delete.go b/daemon/image_delete.go index 4340df8001..8056a87c86 100644 --- a/daemon/image_delete.go +++ b/daemon/image_delete.go @@ -160,7 +160,7 @@ func (daemon *Daemon) canDeleteImage(imgID string, force bool) error { return err } - if err := parent.WalkHistory(func(p *image.Image) error { + if err := daemon.graph.WalkHistory(parent, func(p *image.Image) error { if imgID == p.ID { if container.IsRunning() { if force { diff --git a/graph/graph.go b/graph/graph.go index e95887fab8..b405e1dd87 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -3,12 +3,14 @@ package graph import ( "compress/gzip" "crypto/sha256" + "encoding/json" "fmt" "io" "io/ioutil" "os" "path/filepath" "runtime" + "strconv" "strings" "syscall" "time" @@ -29,7 +31,7 @@ import ( // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string + root string idIndex *truncindex.TruncIndex driver graphdriver.Driver } @@ -47,7 +49,7 @@ func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) { } graph := &Graph{ - Root: abspath, + root: abspath, idIndex: truncindex.NewTruncIndex([]string{}), driver: driver, } @@ -58,7 +60,7 @@ func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) { } func (graph *Graph) restore() error { - dir, err := ioutil.ReadDir(graph.Root) + dir, err := ioutil.ReadDir(graph.root) if err != nil { return err } @@ -95,14 +97,13 @@ func (graph *Graph) Get(name string) (*image.Image, error) { if err != nil { return nil, fmt.Errorf("could not find image: %v", err) } - img, err := image.LoadImage(graph.ImageRoot(id)) + img, err := graph.loadImage(id) if err != nil { return nil, err } if img.ID != id { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID) } - img.SetGraph(graph) if img.Size < 0 { size, err := graph.driver.DiffSize(img.ID, img.Parent) @@ -111,7 +112,7 @@ func (graph *Graph) Get(name string) (*image.Image, error) { } img.Size = size - if err := img.SaveSize(graph.ImageRoot(id)); err != nil { + if err := graph.saveSize(graph.imageRoot(id), int(img.Size)); err != nil { return nil, err } } @@ -164,7 +165,7 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) // Ensure that the image root does not exist on the filesystem // when it is not registered in the graph. // This is common when you switch from one graph driver to another - if err := os.RemoveAll(graph.ImageRoot(img.ID)); err != nil && !os.IsNotExist(err) { + if err := os.RemoveAll(graph.imageRoot(img.ID)); err != nil && !os.IsNotExist(err) { return err } @@ -174,10 +175,10 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) // (FIXME: make that mandatory for drivers). graph.driver.Remove(img.ID) - tmp, err := graph.Mktemp("") + tmp, err := graph.mktemp("") defer os.RemoveAll(tmp) if err != nil { - return fmt.Errorf("Mktemp failed: %s", err) + return fmt.Errorf("mktemp failed: %s", err) } // Create root filesystem in the driver @@ -185,12 +186,11 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err) } // Apply the diff/layer - img.SetGraph(graph) - if err := image.StoreImage(img, layerData, tmp); err != nil { + if err := graph.storeImage(img, layerData, tmp); err != nil { return err } // Commit - if err := os.Rename(tmp, graph.ImageRoot(img.ID)); err != nil { + if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil { return err } graph.idIndex.Add(img.ID) @@ -200,17 +200,16 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) // TempLayerArchive creates a temporary archive of the given image's filesystem layer. // The archive is stored on disk and will be automatically deleted as soon as has been read. // If output is not nil, a human-readable progress bar will be written to it. -// FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives? func (graph *Graph) TempLayerArchive(id string, sf *streamformatter.StreamFormatter, output io.Writer) (*archive.TempArchive, error) { image, err := graph.Get(id) if err != nil { return nil, err } - tmp, err := graph.Mktemp("") + tmp, err := graph.mktemp("") if err != nil { return nil, err } - a, err := image.TarLayer() + a, err := graph.TarLayer(image) if err != nil { return nil, err } @@ -227,9 +226,9 @@ func (graph *Graph) TempLayerArchive(id string, sf *streamformatter.StreamFormat return archive.NewTempArchive(progressReader, tmp) } -// Mktemp creates a temporary sub-directory inside the graph's filesystem. -func (graph *Graph) Mktemp(id string) (string, error) { - dir := filepath.Join(graph.Root, "_tmp", stringid.GenerateRandomID()) +// mktemp creates a temporary sub-directory inside the graph's filesystem. +func (graph *Graph) mktemp(id string) (string, error) { + dir := filepath.Join(graph.root, "_tmp", stringid.GenerateRandomID()) if err := system.MkdirAll(dir, 0700); err != nil { return "", err } @@ -237,7 +236,7 @@ func (graph *Graph) Mktemp(id string) (string, error) { } func (graph *Graph) newTempFile() (*os.File, error) { - tmp, err := graph.Mktemp("") + tmp, err := graph.mktemp("") if err != nil { return nil, err } @@ -342,17 +341,17 @@ func (graph *Graph) Delete(name string) error { if err != nil { return err } - tmp, err := graph.Mktemp("") + tmp, err := graph.mktemp("") graph.idIndex.Delete(id) if err == nil { - if err := os.Rename(graph.ImageRoot(id), tmp); err != nil { + if err := os.Rename(graph.imageRoot(id), tmp); err != nil { // On err make tmp point to old dir and cleanup unused tmp dir os.RemoveAll(tmp) - tmp = graph.ImageRoot(id) + tmp = graph.imageRoot(id) } } else { // On err make tmp point to old dir for cleanup - tmp = graph.ImageRoot(id) + tmp = graph.imageRoot(id) } // Remove rootfs data from the driver graph.driver.Remove(id) @@ -375,7 +374,7 @@ func (graph *Graph) Map() (map[string]*image.Image, error) { // walkAll iterates over each image in the graph, and passes it to a handler. // The walking order is undetermined. func (graph *Graph) walkAll(handler func(*image.Image)) error { - files, err := ioutil.ReadDir(graph.Root) + files, err := ioutil.ReadDir(graph.root) if err != nil { return err } @@ -428,10 +427,125 @@ func (graph *Graph) Heads() (map[string]*image.Image, error) { return heads, err } -func (graph *Graph) ImageRoot(id string) string { - return filepath.Join(graph.Root, id) +func (graph *Graph) imageRoot(id string) string { + return filepath.Join(graph.root, id) } -func (graph *Graph) Driver() graphdriver.Driver { - return graph.driver +// storeImage stores file system layer data for the given image to the +// graph's storage driver. Image metadata is stored in a file +// at the specified root directory. +func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader, root string) (err error) { + // Store the layer. If layerData is not nil, unpack it into the new layer + if layerData != nil { + if img.Size, err = graph.driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil { + return err + } + } + + if err := graph.saveSize(root, int(img.Size)); err != nil { + return err + } + + f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) + if err != nil { + return err + } + + defer f.Close() + + return json.NewEncoder(f).Encode(img) +} + +// loadImage fetches the image with the given id from the graph. +func (graph *Graph) loadImage(id string) (*image.Image, error) { + root := graph.imageRoot(id) + + // Open the JSON file to decode by streaming + jsonSource, err := os.Open(jsonPath(root)) + if err != nil { + return nil, err + } + defer jsonSource.Close() + + img := &image.Image{} + dec := json.NewDecoder(jsonSource) + + // Decode the JSON data + if err := dec.Decode(img); err != nil { + return nil, err + } + if err := image.ValidateID(img.ID); err != nil { + return nil, err + } + + if buf, err := ioutil.ReadFile(filepath.Join(root, "layersize")); err != nil { + if !os.IsNotExist(err) { + return nil, err + } + // If the layersize file does not exist then set the size to a negative number + // because a layer size of 0 (zero) is valid + img.Size = -1 + } else { + // Using Atoi here instead would temporarily convert the size to a machine + // dependent integer type, which causes images larger than 2^31 bytes to + // display negative sizes on 32-bit machines: + size, err := strconv.ParseInt(string(buf), 10, 64) + if err != nil { + return nil, err + } + img.Size = int64(size) + } + + return img, nil +} + +// saveSize stores the `size` in the provided graph `img` directory `root`. +func (graph *Graph) saveSize(root string, size int) error { + if err := ioutil.WriteFile(filepath.Join(root, "layersize"), []byte(strconv.Itoa(size)), 0600); err != nil { + return fmt.Errorf("Error storing image size in %s/layersize: %s", root, err) + } + return nil +} + +// SetCheckSum sets the checksum for the image layer to the provided value. +func (graph *Graph) SetCheckSum(id, checksum string) error { + root := graph.imageRoot(id) + if err := ioutil.WriteFile(filepath.Join(root, "checksum"), []byte(checksum), 0600); err != nil { + return fmt.Errorf("Error storing checksum in %s/checksum: %s", root, err) + } + return nil +} + +// GetCheckSum gets the checksum for the provide image layer id. +func (graph *Graph) GetCheckSum(id string) (string, error) { + root := graph.imageRoot(id) + cs, err := ioutil.ReadFile(filepath.Join(root, "checksum")) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", err + } + return string(cs), err +} + +// RawJSON returns the JSON representation for an image as a byte array. +func (graph *Graph) RawJSON(id string) ([]byte, error) { + root := graph.imageRoot(id) + + buf, err := ioutil.ReadFile(jsonPath(root)) + if err != nil { + return nil, fmt.Errorf("Failed to read json for image %s: %s", id, err) + } + + return buf, nil +} + +func jsonPath(root string) string { + return filepath.Join(root, "json") +} + +// TarLayer returns a tar archive of the image's filesystem layer. +func (graph *Graph) TarLayer(img *image.Image) (arch archive.Archive, err error) { + return graph.driver.Diff(img.ID, img.Parent) } diff --git a/graph/graph_test.go b/graph/graph_test.go index 81471b6749..71c84b0894 100644 --- a/graph/graph_test.go +++ b/graph/graph_test.go @@ -17,7 +17,7 @@ import ( func TestMount(t *testing.T) { graph, driver := tempGraph(t) - defer os.RemoveAll(graph.Root) + defer os.RemoveAll(graph.root) defer driver.Cleanup() archive, err := fakeTar() @@ -52,7 +52,7 @@ func TestInit(t *testing.T) { graph, _ := tempGraph(t) defer nukeGraph(graph) // Root should exist - if _, err := os.Stat(graph.Root); err != nil { + if _, err := os.Stat(graph.root); err != nil { t.Fatal(err) } // Map() should be empty @@ -301,6 +301,6 @@ func tempGraph(t *testing.T) (*Graph, graphdriver.Driver) { } func nukeGraph(graph *Graph) { - graph.Driver().Cleanup() - os.RemoveAll(graph.Root) + graph.driver.Cleanup() + os.RemoveAll(graph.root) } diff --git a/graph/history.go b/graph/history.go index 56e759a8eb..280e7c8358 100644 --- a/graph/history.go +++ b/graph/history.go @@ -1,6 +1,7 @@ package graph import ( + "fmt" "strings" "github.com/docker/docker/api/types" @@ -8,6 +9,78 @@ import ( "github.com/docker/docker/utils" ) +// History returns the list of all images used to create this image. +func (graph *Graph) History(img *image.Image) ([]*image.Image, error) { + var parents []*image.Image + if err := graph.WalkHistory(img, + func(img *image.Image) error { + parents = append(parents, img) + return nil + }, + ); err != nil { + return nil, err + } + return parents, nil +} + +// WalkHistory calls the handler function for each image in the +// provided images lineage starting from immediate parent. +func (graph *Graph) WalkHistory(img *image.Image, handler func(*image.Image) error) (err error) { + currentImg := img + for currentImg != nil { + if handler != nil { + if err := handler(currentImg); err != nil { + return err + } + } + currentImg, err = graph.GetParent(currentImg) + if err != nil { + return fmt.Errorf("Error while getting parent image: %v", err) + } + } + return nil +} + +// depth returns the number of parents for a +// current image +func (graph *Graph) depth(img *image.Image) (int, error) { + var ( + count = 0 + parent = img + err error + ) + + for parent != nil { + count++ + parent, err = graph.GetParent(parent) + if err != nil { + return -1, err + } + } + return count, nil +} + +// Set the max depth to the aufs default that most +// kernels are compiled with +// For more information see: http://sourceforge.net/p/aufs/aufs3-standalone/ci/aufs3.12/tree/config.mk +const MaxImageDepth = 127 + +// CheckDepth returns an error if the depth of an image, as returned +// by ImageDepth, is too large to support creating a container from it +// on this daemon. +func (graph *Graph) CheckDepth(img *image.Image) error { + // We add 2 layers to the depth because the container's rw and + // init layer add to the restriction + depth, err := graph.depth(img) + if err != nil { + return err + } + if depth+2 >= MaxImageDepth { + return fmt.Errorf("Cannot create container with more than %d parents", MaxImageDepth) + } + return nil +} + func (s *TagStore) History(name string) ([]*types.ImageHistory, error) { foundImage, err := s.LookupImage(name) if err != nil { @@ -27,7 +100,7 @@ func (s *TagStore) History(name string) ([]*types.ImageHistory, error) { history := []*types.ImageHistory{} - err = foundImage.WalkHistory(func(img *image.Image) error { + err = s.graph.WalkHistory(foundImage, func(img *image.Image) error { history = append(history, &types.ImageHistory{ ID: img.ID, Created: img.Created.Unix(), @@ -41,3 +114,19 @@ func (s *TagStore) History(name string) ([]*types.ImageHistory, error) { return history, err } + +func (graph *Graph) GetParent(img *image.Image) (*image.Image, error) { + if img.Parent == "" { + return nil, nil + } + return graph.Get(img.Parent) +} + +func (graph *Graph) GetParentsSize(img *image.Image, size int64) int64 { + parentImage, err := graph.GetParent(img) + if err != nil || parentImage == nil { + return size + } + size += parentImage.Size + return graph.GetParentsSize(parentImage, size) +} diff --git a/graph/list.go b/graph/list.go index f95508e950..f5f51f68b9 100644 --- a/graph/list.go +++ b/graph/list.go @@ -103,7 +103,7 @@ func (s *TagStore) Images(config *ImagesConfig) ([]*types.Image, error) { newImage.ID = image.ID newImage.Created = int(image.Created.Unix()) newImage.Size = int(image.Size) - newImage.VirtualSize = int(image.GetParentsSize(0) + image.Size) + newImage.VirtualSize = int(s.graph.GetParentsSize(image, 0) + image.Size) newImage.Labels = image.ContainerConfig.Labels if utils.DigestReference(ref) { @@ -140,7 +140,7 @@ func (s *TagStore) Images(config *ImagesConfig) ([]*types.Image, error) { newImage.ID = image.ID newImage.Created = int(image.Created.Unix()) newImage.Size = int(image.Size) - newImage.VirtualSize = int(image.GetParentsSize(0) + image.Size) + newImage.VirtualSize = int(s.graph.GetParentsSize(image, 0) + image.Size) newImage.Labels = image.ContainerConfig.Labels images = append(images, newImage) diff --git a/graph/manifest_test.go b/graph/manifest_test.go index 63086f4d55..0b8e7a2fb0 100644 --- a/graph/manifest_test.go +++ b/graph/manifest_test.go @@ -57,7 +57,7 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error metadata = *layer.Config } - for ; layer != nil; layer, err = layer.GetParent() { + for ; layer != nil; layer, err = s.graph.GetParent(layer) { if err != nil { return nil, err } @@ -72,12 +72,12 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error } } - checksum, err := layer.GetCheckSum(s.graph.ImageRoot(layer.ID)) + checksum, err := s.graph.GetCheckSum(layer.ID) if err != nil { return nil, fmt.Errorf("Error getting image checksum: %s", err) } if tarsum.VersionLabelForChecksum(checksum) != tarsum.Version1.String() { - archive, err := layer.TarLayer() + archive, err := s.graph.TarLayer(layer) if err != nil { return nil, err } @@ -95,12 +95,12 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error checksum = tarSum.Sum(nil) // Save checksum value - if err := layer.SaveCheckSum(s.graph.ImageRoot(layer.ID), checksum); err != nil { + if err := s.graph.SetCheckSum(layer.ID, checksum); err != nil { return nil, err } } - jsonData, err := layer.RawJson() + jsonData, err := s.graph.RawJSON(layer.ID) if err != nil { return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err) } @@ -141,7 +141,7 @@ func TestManifestTarsumCache(t *testing.T) { t.Fatal(err) } - if cs, err := img.GetCheckSum(store.graph.ImageRoot(testManifestImageID)); err != nil { + if cs, err := store.graph.GetCheckSum(testManifestImageID); err != nil { t.Fatal(err) } else if cs != "" { t.Fatalf("Non-empty checksum file after register") @@ -153,7 +153,7 @@ func TestManifestTarsumCache(t *testing.T) { t.Fatal(err) } - manifestChecksum, err := img.GetCheckSum(store.graph.ImageRoot(testManifestImageID)) + manifestChecksum, err := store.graph.GetCheckSum(testManifestImageID) if err != nil { t.Fatal(err) } @@ -175,7 +175,7 @@ func TestManifestTarsumCache(t *testing.T) { t.Fatalf("Unexpected number of layer history, expecting 1: %d", len(manifest.History)) } - v1compat, err := img.RawJson() + v1compat, err := store.graph.RawJSON(img.ID) if err != nil { t.Fatal(err) } @@ -207,7 +207,7 @@ func TestManifestDigestCheck(t *testing.T) { t.Fatal(err) } - if cs, err := img.GetCheckSum(store.graph.ImageRoot(testManifestImageID)); err != nil { + if cs, err := store.graph.GetCheckSum(testManifestImageID); err != nil { t.Fatal(err) } else if cs != "" { t.Fatalf("Non-empty checksum file after register") diff --git a/graph/push.go b/graph/push.go index 532256fb54..24526b4507 100644 --- a/graph/push.go +++ b/graph/push.go @@ -5,9 +5,7 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" - "path/filepath" "sync" "github.com/Sirupsen/logrus" @@ -57,7 +55,7 @@ func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string tagsByImage[id] = append(tagsByImage[id], tag) - for img, err := s.graph.Get(id); img != nil; img, err = img.GetParent() { + for img, err := s.graph.Get(id); img != nil; img, err = s.graph.GetParent(img) { if err != nil { return nil, nil, err } @@ -248,7 +246,7 @@ func (s *TagStore) pushRepository(r *registry.Session, out io.Writer, func (s *TagStore) pushImage(r *registry.Session, out io.Writer, imgID, ep string, token []string, sf *streamformatter.StreamFormatter) (checksum string, err error) { out = ioutils.NewWriteFlusher(out) - jsonRaw, err := ioutil.ReadFile(filepath.Join(s.graph.Root, imgID, "json")) + jsonRaw, err := s.graph.RawJSON(imgID) if err != nil { return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) } @@ -349,7 +347,7 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o layersSeen := make(map[string]bool) layers := []*image.Image{layer} - for ; layer != nil; layer, err = layer.GetParent() { + for ; layer != nil; layer, err = s.graph.GetParent(layer) { if err != nil { return err } @@ -372,12 +370,12 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o return err } } - jsonData, err := layer.RawJson() + jsonData, err := s.graph.RawJSON(layer.ID) if err != nil { return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err) } - checksum, err := layer.GetCheckSum(s.graph.ImageRoot(layer.ID)) + checksum, err := s.graph.GetCheckSum(layer.ID) if err != nil { return fmt.Errorf("error getting image checksum: %s", err) } @@ -401,7 +399,7 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o return err } else if cs != checksum { // Cache new checksum - if err := layer.SaveCheckSum(s.graph.ImageRoot(layer.ID), cs); err != nil { + if err := s.graph.SetCheckSum(layer.ID, cs); err != nil { return err } checksum = cs @@ -456,7 +454,7 @@ func (s *TagStore) pushV2Image(r *registry.Session, img *image.Image, endpoint * if err != nil { return "", err } - arch, err := image.TarLayer() + arch, err := s.graph.TarLayer(image) if err != nil { return "", err } diff --git a/graph/service.go b/graph/service.go index 52dde1d980..7718bab769 100644 --- a/graph/service.go +++ b/graph/service.go @@ -14,7 +14,7 @@ func (s *TagStore) LookupRaw(name string) ([]byte, error) { return nil, fmt.Errorf("No such image %s", name) } - imageInspectRaw, err := image.RawJson() + imageInspectRaw, err := s.graph.RawJSON(image.ID) if err != nil { return nil, err } @@ -42,7 +42,7 @@ func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) { Architecture: image.Architecture, Os: image.OS, Size: image.Size, - VirtualSize: image.GetParentsSize(0) + image.Size, + VirtualSize: s.graph.GetParentsSize(image, 0) + image.Size, } return imageInspect, nil @@ -51,7 +51,7 @@ func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) { // ImageTarLayer return the tarLayer of the image func (s *TagStore) ImageTarLayer(name string, dest io.Writer) error { if image, err := s.LookupImage(name); err == nil && image != nil { - fs, err := image.TarLayer() + fs, err := s.graph.TarLayer(image) if err != nil { return err } diff --git a/image/graph.go b/image/graph.go deleted file mode 100644 index 31fbdd9ad8..0000000000 --- a/image/graph.go +++ /dev/null @@ -1,11 +0,0 @@ -package image - -import ( - "github.com/docker/docker/daemon/graphdriver" -) - -type Graph interface { - Get(id string) (*Image, error) - ImageRoot(id string) string - Driver() graphdriver.Driver -} diff --git a/image/image.go b/image/image.go index 4e37ebc42d..218f18f2d7 100644 --- a/image/image.go +++ b/image/image.go @@ -3,22 +3,12 @@ package image import ( "encoding/json" "fmt" - "io/ioutil" - "os" - "path/filepath" "regexp" - "strconv" "time" - "github.com/docker/docker/pkg/archive" "github.com/docker/docker/runconfig" ) -// Set the max depth to the aufs default that most -// kernels are compiled with -// For more information see: http://sourceforge.net/p/aufs/aufs3-standalone/ci/aufs3.12/tree/config.mk -const MaxImageDepth = 127 - var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) type Image struct { @@ -34,225 +24,6 @@ type Image struct { Architecture string `json:"architecture,omitempty"` OS string `json:"os,omitempty"` Size int64 - - graph Graph -} - -func LoadImage(root string) (*Image, error) { - // Open the JSON file to decode by streaming - jsonSource, err := os.Open(jsonPath(root)) - if err != nil { - return nil, err - } - defer jsonSource.Close() - - img := &Image{} - dec := json.NewDecoder(jsonSource) - - // Decode the JSON data - if err := dec.Decode(img); err != nil { - return nil, err - } - if err := ValidateID(img.ID); err != nil { - return nil, err - } - - if buf, err := ioutil.ReadFile(filepath.Join(root, "layersize")); err != nil { - if !os.IsNotExist(err) { - return nil, err - } - // If the layersize file does not exist then set the size to a negative number - // because a layer size of 0 (zero) is valid - img.Size = -1 - } else { - // Using Atoi here instead would temporarily convert the size to a machine - // dependent integer type, which causes images larger than 2^31 bytes to - // display negative sizes on 32-bit machines: - size, err := strconv.ParseInt(string(buf), 10, 64) - if err != nil { - return nil, err - } - img.Size = int64(size) - } - - return img, nil -} - -// StoreImage stores file system layer data for the given image to the -// image's registered storage driver. Image metadata is stored in a file -// at the specified root directory. -func StoreImage(img *Image, layerData archive.ArchiveReader, root string) (err error) { - // Store the layer. If layerData is not nil, unpack it into the new layer - if layerData != nil { - if img.Size, err = img.graph.Driver().ApplyDiff(img.ID, img.Parent, layerData); err != nil { - return err - } - } - - if err := img.SaveSize(root); err != nil { - return err - } - - f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) - if err != nil { - return err - } - - defer f.Close() - - return json.NewEncoder(f).Encode(img) -} - -func (img *Image) SetGraph(graph Graph) { - img.graph = graph -} - -// SaveSize stores the current `size` value of `img` in the directory `root`. -func (img *Image) SaveSize(root string) error { - if err := ioutil.WriteFile(filepath.Join(root, "layersize"), []byte(strconv.Itoa(int(img.Size))), 0600); err != nil { - return fmt.Errorf("Error storing image size in %s/layersize: %s", root, err) - } - return nil -} - -func (img *Image) SaveCheckSum(root, checksum string) error { - if err := ioutil.WriteFile(filepath.Join(root, "checksum"), []byte(checksum), 0600); err != nil { - return fmt.Errorf("Error storing checksum in %s/checksum: %s", root, err) - } - return nil -} - -func (img *Image) GetCheckSum(root string) (string, error) { - cs, err := ioutil.ReadFile(filepath.Join(root, "checksum")) - if err != nil { - if os.IsNotExist(err) { - return "", nil - } - return "", err - } - return string(cs), err -} - -func jsonPath(root string) string { - return filepath.Join(root, "json") -} - -func (img *Image) RawJson() ([]byte, error) { - root, err := img.root() - if err != nil { - return nil, fmt.Errorf("Failed to get root for image %s: %s", img.ID, err) - } - - buf, err := ioutil.ReadFile(jsonPath(root)) - if err != nil { - return nil, fmt.Errorf("Failed to read json for image %s: %s", img.ID, err) - } - - return buf, nil -} - -// TarLayer returns a tar archive of the image's filesystem layer. -func (img *Image) TarLayer() (arch archive.Archive, err error) { - if img.graph == nil { - return nil, fmt.Errorf("Can't load storage driver for unregistered image %s", img.ID) - } - - driver := img.graph.Driver() - - return driver.Diff(img.ID, img.Parent) -} - -// Image includes convenience proxy functions to its graph -// These functions will return an error if the image is not registered -// (ie. if image.graph == nil) -func (img *Image) History() ([]*Image, error) { - var parents []*Image - if err := img.WalkHistory( - func(img *Image) error { - parents = append(parents, img) - return nil - }, - ); err != nil { - return nil, err - } - return parents, nil -} - -func (img *Image) WalkHistory(handler func(*Image) error) (err error) { - currentImg := img - for currentImg != nil { - if handler != nil { - if err := handler(currentImg); err != nil { - return err - } - } - currentImg, err = currentImg.GetParent() - if err != nil { - return fmt.Errorf("Error while getting parent image: %v", err) - } - } - return nil -} - -func (img *Image) GetParent() (*Image, error) { - if img.Parent == "" { - return nil, nil - } - if img.graph == nil { - return nil, fmt.Errorf("Can't lookup parent of unregistered image") - } - return img.graph.Get(img.Parent) -} - -func (img *Image) root() (string, error) { - if img.graph == nil { - return "", fmt.Errorf("Can't lookup root of unregistered image") - } - return img.graph.ImageRoot(img.ID), nil -} - -func (img *Image) GetParentsSize(size int64) int64 { - parentImage, err := img.GetParent() - if err != nil || parentImage == nil { - return size - } - size += parentImage.Size - return parentImage.GetParentsSize(size) -} - -// Depth returns the number of parents for a -// current image -func (img *Image) Depth() (int, error) { - var ( - count = 0 - parent = img - err error - ) - - for parent != nil { - count++ - parent, err = parent.GetParent() - if err != nil { - return -1, err - } - } - return count, nil -} - -// CheckDepth returns an error if the depth of an image, as returned -// by ImageDepth, is too large to support creating a container from it -// on this daemon. -func (img *Image) CheckDepth() error { - // We add 2 layers to the depth because the container's rw and - // init layer add to the restriction - depth, err := img.Depth() - if err != nil { - return err - } - if depth+2 >= MaxImageDepth { - return fmt.Errorf("Cannot create container with more than %d parents", MaxImageDepth) - } - return nil } // Build an Image object from raw json data From bb50a4159bda8e6f2d47b69ddd476ab32ac6be14 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 5 Jun 2015 15:32:31 -0700 Subject: [PATCH 2/3] Update graph walkhistory to pass by value Remove unused graph history function Signed-off-by: Derek McGowan (github: dmcgowan) --- daemon/image_delete.go | 2 +- graph/history.go | 20 +++----------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/daemon/image_delete.go b/daemon/image_delete.go index 8056a87c86..d0e7e20d4b 100644 --- a/daemon/image_delete.go +++ b/daemon/image_delete.go @@ -160,7 +160,7 @@ func (daemon *Daemon) canDeleteImage(imgID string, force bool) error { return err } - if err := daemon.graph.WalkHistory(parent, func(p *image.Image) error { + if err := daemon.graph.WalkHistory(parent, func(p image.Image) error { if imgID == p.ID { if container.IsRunning() { if force { diff --git a/graph/history.go b/graph/history.go index 280e7c8358..4bb93fc3f9 100644 --- a/graph/history.go +++ b/graph/history.go @@ -9,27 +9,13 @@ import ( "github.com/docker/docker/utils" ) -// History returns the list of all images used to create this image. -func (graph *Graph) History(img *image.Image) ([]*image.Image, error) { - var parents []*image.Image - if err := graph.WalkHistory(img, - func(img *image.Image) error { - parents = append(parents, img) - return nil - }, - ); err != nil { - return nil, err - } - return parents, nil -} - // WalkHistory calls the handler function for each image in the // provided images lineage starting from immediate parent. -func (graph *Graph) WalkHistory(img *image.Image, handler func(*image.Image) error) (err error) { +func (graph *Graph) WalkHistory(img *image.Image, handler func(image.Image) error) (err error) { currentImg := img for currentImg != nil { if handler != nil { - if err := handler(currentImg); err != nil { + if err := handler(*currentImg); err != nil { return err } } @@ -100,7 +86,7 @@ func (s *TagStore) History(name string) ([]*types.ImageHistory, error) { history := []*types.ImageHistory{} - err = s.graph.WalkHistory(foundImage, func(img *image.Image) error { + err = s.graph.WalkHistory(foundImage, func(img image.Image) error { history = append(history, &types.ImageHistory{ ID: img.ID, Created: img.Created.Unix(), From c0b44218196ab597d22eb6344e8770fdee73067b Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 5 Jun 2015 18:07:41 -0700 Subject: [PATCH 3/3] Update graph to use digest type Update get and set functions to use digests. Update push code to use the digest type instead of string Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/graph.go | 24 ++++++++++++++++-------- graph/manifest_test.go | 39 +++++++++++++++------------------------ graph/push.go | 29 ++++++++++++----------------- 3 files changed, 43 insertions(+), 49 deletions(-) diff --git a/graph/graph.go b/graph/graph.go index b405e1dd87..5ccb66c666 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -4,6 +4,7 @@ import ( "compress/gzip" "crypto/sha256" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -36,6 +37,13 @@ type Graph struct { driver graphdriver.Driver } +var ( + // ErrDigestNotSet is used when request the digest for a layer + // but the layer has no digest value or content to compute the + // the digest. + ErrDigestNotSet = errors.New("digest is not set for layer") +) + // NewGraph instantiates a new graph at the given root path in the filesystem. // `root` will be created if it doesn't exist. func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) { @@ -507,26 +515,26 @@ func (graph *Graph) saveSize(root string, size int) error { return nil } -// SetCheckSum sets the checksum for the image layer to the provided value. -func (graph *Graph) SetCheckSum(id, checksum string) error { +// SetDigest sets the digest for the image layer to the provided value. +func (graph *Graph) SetDigest(id string, dgst digest.Digest) error { root := graph.imageRoot(id) - if err := ioutil.WriteFile(filepath.Join(root, "checksum"), []byte(checksum), 0600); err != nil { - return fmt.Errorf("Error storing checksum in %s/checksum: %s", root, err) + if err := ioutil.WriteFile(filepath.Join(root, "checksum"), []byte(dgst.String()), 0600); err != nil { + return fmt.Errorf("Error storing digest in %s/checksum: %s", root, err) } return nil } -// GetCheckSum gets the checksum for the provide image layer id. -func (graph *Graph) GetCheckSum(id string) (string, error) { +// GetDigest gets the digest for the provide image layer id. +func (graph *Graph) GetDigest(id string) (digest.Digest, error) { root := graph.imageRoot(id) cs, err := ioutil.ReadFile(filepath.Join(root, "checksum")) if err != nil { if os.IsNotExist(err) { - return "", nil + return "", ErrDigestNotSet } return "", err } - return string(cs), err + return digest.ParseDigest(string(cs)) } // RawJSON returns the JSON representation for an image as a byte array. diff --git a/graph/manifest_test.go b/graph/manifest_test.go index 0b8e7a2fb0..6b1a608073 100644 --- a/graph/manifest_test.go +++ b/graph/manifest_test.go @@ -3,14 +3,11 @@ package graph import ( "encoding/json" "fmt" - "io" - "io/ioutil" "os" "testing" "github.com/docker/distribution/digest" "github.com/docker/docker/image" - "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" "github.com/docker/docker/utils" @@ -72,11 +69,8 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error } } - checksum, err := s.graph.GetCheckSum(layer.ID) - if err != nil { - return nil, fmt.Errorf("Error getting image checksum: %s", err) - } - if tarsum.VersionLabelForChecksum(checksum) != tarsum.Version1.String() { + dgst, err := s.graph.GetDigest(layer.ID) + if err == ErrDigestNotSet { archive, err := s.graph.TarLayer(layer) if err != nil { return nil, err @@ -84,20 +78,17 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error defer archive.Close() - tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version1) + dgst, err = digest.FromReader(archive) if err != nil { return nil, err } - if _, err := io.Copy(ioutil.Discard, tarSum); err != nil { - return nil, err - } - - checksum = tarSum.Sum(nil) // Save checksum value - if err := s.graph.SetCheckSum(layer.ID, checksum); err != nil { + if err := s.graph.SetDigest(layer.ID, dgst); err != nil { return nil, err } + } else if err != nil { + return nil, fmt.Errorf("Error getting image checksum: %s", err) } jsonData, err := s.graph.RawJSON(layer.ID) @@ -105,7 +96,7 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err) } - manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: checksum}) + manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: dgst.String()}) layersSeen[layer.ID] = true @@ -141,10 +132,10 @@ func TestManifestTarsumCache(t *testing.T) { t.Fatal(err) } - if cs, err := store.graph.GetCheckSum(testManifestImageID); err != nil { - t.Fatal(err) - } else if cs != "" { + if _, err := store.graph.GetDigest(testManifestImageID); err == nil { t.Fatalf("Non-empty checksum file after register") + } else if err != ErrDigestNotSet { + t.Fatal(err) } // Generate manifest @@ -153,7 +144,7 @@ func TestManifestTarsumCache(t *testing.T) { t.Fatal(err) } - manifestChecksum, err := store.graph.GetCheckSum(testManifestImageID) + manifestChecksum, err := store.graph.GetDigest(testManifestImageID) if err != nil { t.Fatal(err) } @@ -167,7 +158,7 @@ func TestManifestTarsumCache(t *testing.T) { t.Fatalf("Unexpected number of layers, expecting 1: %d", len(manifest.FSLayers)) } - if manifest.FSLayers[0].BlobSum != manifestChecksum { + if manifest.FSLayers[0].BlobSum != manifestChecksum.String() { t.Fatalf("Unexpected blob sum, expecting %q, got %q", manifestChecksum, manifest.FSLayers[0].BlobSum) } @@ -207,10 +198,10 @@ func TestManifestDigestCheck(t *testing.T) { t.Fatal(err) } - if cs, err := store.graph.GetCheckSum(testManifestImageID); err != nil { - t.Fatal(err) - } else if cs != "" { + if _, err := store.graph.GetDigest(testManifestImageID); err == nil { t.Fatalf("Non-empty checksum file after register") + } else if err != ErrDigestNotSet { + t.Fatal(err) } // Generate manifest diff --git a/graph/push.go b/graph/push.go index 24526b4507..0de40b8c50 100644 --- a/graph/push.go +++ b/graph/push.go @@ -375,18 +375,13 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err) } - checksum, err := s.graph.GetCheckSum(layer.ID) - if err != nil { - return fmt.Errorf("error getting image checksum: %s", err) - } - var exists bool - if len(checksum) > 0 { - dgst, err := digest.ParseDigest(checksum) - if err != nil { - return fmt.Errorf("Invalid checksum %s: %s", checksum, err) + dgst, err := s.graph.GetDigest(layer.ID) + if err != nil { + if err != ErrDigestNotSet { + return fmt.Errorf("error getting image checksum: %s", err) } - + } else { // Call mount blob exists, err = r.HeadV2ImageBlob(endpoint, repoInfo.RemoteName, dgst, auth) if err != nil { @@ -395,19 +390,19 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o } } if !exists { - if cs, err := s.pushV2Image(r, layer, endpoint, repoInfo.RemoteName, sf, out, auth); err != nil { + if pushDigest, err := s.pushV2Image(r, layer, endpoint, repoInfo.RemoteName, sf, out, auth); err != nil { return err - } else if cs != checksum { + } else if pushDigest != dgst { // Cache new checksum - if err := s.graph.SetCheckSum(layer.ID, cs); err != nil { + if err := s.graph.SetDigest(layer.ID, pushDigest); err != nil { return err } - checksum = cs + dgst = pushDigest } } else { out.Write(sf.FormatProgress(stringid.TruncateID(layer.ID), "Image already exists", nil)) } - m.FSLayers[i] = ®istry.FSLayer{BlobSum: checksum} + m.FSLayers[i] = ®istry.FSLayer{BlobSum: dgst.String()} m.History[i] = ®istry.ManifestHistory{V1Compatibility: string(jsonData)} } @@ -447,7 +442,7 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o } // PushV2Image pushes the image content to the v2 registry, first buffering the contents to disk -func (s *TagStore) pushV2Image(r *registry.Session, img *image.Image, endpoint *registry.Endpoint, imageName string, sf *streamformatter.StreamFormatter, out io.Writer, auth *registry.RequestAuthorization) (string, error) { +func (s *TagStore) pushV2Image(r *registry.Session, img *image.Image, endpoint *registry.Endpoint, imageName string, sf *streamformatter.StreamFormatter, out io.Writer, auth *registry.RequestAuthorization) (digest.Digest, error) { out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Buffering to Disk", nil)) image, err := s.graph.Get(img.ID) @@ -488,7 +483,7 @@ func (s *TagStore) pushV2Image(r *registry.Session, img *image.Image, endpoint * return "", err } out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Image successfully pushed", nil)) - return dgst.String(), nil + return dgst, nil } // FIXME: Allow to interrupt current push when new push of same image is done.