Merge pull request #13773 from dmcgowan/refactor-1-image-graph-separation

refactor: separate graph from image
This commit is contained in:
Arnaud Porterie 2015-06-11 17:44:37 -07:00
commit 00b8fec75f
11 changed files with 273 additions and 332 deletions

View File

@ -63,7 +63,7 @@ func (daemon *Daemon) Create(config *runconfig.Config, hostConfig *runconfig.Hos
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
if err = img.CheckDepth(); err != nil { if err = daemon.graph.CheckDepth(img); err != nil {
return nil, nil, err return nil, nil, err
} }
imgID = img.ID imgID = img.ID

View File

@ -160,7 +160,7 @@ func (daemon *Daemon) canDeleteImage(imgID string, force bool) error {
return err return err
} }
if err := parent.WalkHistory(func(p *image.Image) error { if err := daemon.graph.WalkHistory(parent, func(p image.Image) error {
if imgID == p.ID { if imgID == p.ID {
if container.IsRunning() { if container.IsRunning() {
if force { if force {

View File

@ -3,12 +3,15 @@ package graph
import ( import (
"compress/gzip" "compress/gzip"
"crypto/sha256" "crypto/sha256"
"encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
"strconv"
"strings" "strings"
"syscall" "syscall"
"time" "time"
@ -29,11 +32,18 @@ import (
// A Graph is a store for versioned filesystem images and the relationship between them. // A Graph is a store for versioned filesystem images and the relationship between them.
type Graph struct { type Graph struct {
Root string root string
idIndex *truncindex.TruncIndex idIndex *truncindex.TruncIndex
driver graphdriver.Driver driver graphdriver.Driver
} }
var (
// ErrDigestNotSet is used when request the digest for a layer
// but the layer has no digest value or content to compute the
// the digest.
ErrDigestNotSet = errors.New("digest is not set for layer")
)
// NewGraph instantiates a new graph at the given root path in the filesystem. // NewGraph instantiates a new graph at the given root path in the filesystem.
// `root` will be created if it doesn't exist. // `root` will be created if it doesn't exist.
func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) { func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
@ -47,7 +57,7 @@ func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
} }
graph := &Graph{ graph := &Graph{
Root: abspath, root: abspath,
idIndex: truncindex.NewTruncIndex([]string{}), idIndex: truncindex.NewTruncIndex([]string{}),
driver: driver, driver: driver,
} }
@ -58,7 +68,7 @@ func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
} }
func (graph *Graph) restore() error { func (graph *Graph) restore() error {
dir, err := ioutil.ReadDir(graph.Root) dir, err := ioutil.ReadDir(graph.root)
if err != nil { if err != nil {
return err return err
} }
@ -95,14 +105,13 @@ func (graph *Graph) Get(name string) (*image.Image, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("could not find image: %v", err) return nil, fmt.Errorf("could not find image: %v", err)
} }
img, err := image.LoadImage(graph.ImageRoot(id)) img, err := graph.loadImage(id)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if img.ID != id { if img.ID != id {
return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID) return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID)
} }
img.SetGraph(graph)
if img.Size < 0 { if img.Size < 0 {
size, err := graph.driver.DiffSize(img.ID, img.Parent) size, err := graph.driver.DiffSize(img.ID, img.Parent)
@ -111,7 +120,7 @@ func (graph *Graph) Get(name string) (*image.Image, error) {
} }
img.Size = size img.Size = size
if err := img.SaveSize(graph.ImageRoot(id)); err != nil { if err := graph.saveSize(graph.imageRoot(id), int(img.Size)); err != nil {
return nil, err return nil, err
} }
} }
@ -164,7 +173,7 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader)
// Ensure that the image root does not exist on the filesystem // Ensure that the image root does not exist on the filesystem
// when it is not registered in the graph. // when it is not registered in the graph.
// This is common when you switch from one graph driver to another // This is common when you switch from one graph driver to another
if err := os.RemoveAll(graph.ImageRoot(img.ID)); err != nil && !os.IsNotExist(err) { if err := os.RemoveAll(graph.imageRoot(img.ID)); err != nil && !os.IsNotExist(err) {
return err return err
} }
@ -174,10 +183,10 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader)
// (FIXME: make that mandatory for drivers). // (FIXME: make that mandatory for drivers).
graph.driver.Remove(img.ID) graph.driver.Remove(img.ID)
tmp, err := graph.Mktemp("") tmp, err := graph.mktemp("")
defer os.RemoveAll(tmp) defer os.RemoveAll(tmp)
if err != nil { if err != nil {
return fmt.Errorf("Mktemp failed: %s", err) return fmt.Errorf("mktemp failed: %s", err)
} }
// Create root filesystem in the driver // Create root filesystem in the driver
@ -185,12 +194,11 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader)
return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err) return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
} }
// Apply the diff/layer // Apply the diff/layer
img.SetGraph(graph) if err := graph.storeImage(img, layerData, tmp); err != nil {
if err := image.StoreImage(img, layerData, tmp); err != nil {
return err return err
} }
// Commit // Commit
if err := os.Rename(tmp, graph.ImageRoot(img.ID)); err != nil { if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil {
return err return err
} }
graph.idIndex.Add(img.ID) graph.idIndex.Add(img.ID)
@ -200,17 +208,16 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader)
// TempLayerArchive creates a temporary archive of the given image's filesystem layer. // TempLayerArchive creates a temporary archive of the given image's filesystem layer.
// The archive is stored on disk and will be automatically deleted as soon as has been read. // The archive is stored on disk and will be automatically deleted as soon as has been read.
// If output is not nil, a human-readable progress bar will be written to it. // If output is not nil, a human-readable progress bar will be written to it.
// FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives?
func (graph *Graph) TempLayerArchive(id string, sf *streamformatter.StreamFormatter, output io.Writer) (*archive.TempArchive, error) { func (graph *Graph) TempLayerArchive(id string, sf *streamformatter.StreamFormatter, output io.Writer) (*archive.TempArchive, error) {
image, err := graph.Get(id) image, err := graph.Get(id)
if err != nil { if err != nil {
return nil, err return nil, err
} }
tmp, err := graph.Mktemp("") tmp, err := graph.mktemp("")
if err != nil { if err != nil {
return nil, err return nil, err
} }
a, err := image.TarLayer() a, err := graph.TarLayer(image)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -227,9 +234,9 @@ func (graph *Graph) TempLayerArchive(id string, sf *streamformatter.StreamFormat
return archive.NewTempArchive(progressReader, tmp) return archive.NewTempArchive(progressReader, tmp)
} }
// Mktemp creates a temporary sub-directory inside the graph's filesystem. // mktemp creates a temporary sub-directory inside the graph's filesystem.
func (graph *Graph) Mktemp(id string) (string, error) { func (graph *Graph) mktemp(id string) (string, error) {
dir := filepath.Join(graph.Root, "_tmp", stringid.GenerateRandomID()) dir := filepath.Join(graph.root, "_tmp", stringid.GenerateRandomID())
if err := system.MkdirAll(dir, 0700); err != nil { if err := system.MkdirAll(dir, 0700); err != nil {
return "", err return "", err
} }
@ -237,7 +244,7 @@ func (graph *Graph) Mktemp(id string) (string, error) {
} }
func (graph *Graph) newTempFile() (*os.File, error) { func (graph *Graph) newTempFile() (*os.File, error) {
tmp, err := graph.Mktemp("") tmp, err := graph.mktemp("")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -342,17 +349,17 @@ func (graph *Graph) Delete(name string) error {
if err != nil { if err != nil {
return err return err
} }
tmp, err := graph.Mktemp("") tmp, err := graph.mktemp("")
graph.idIndex.Delete(id) graph.idIndex.Delete(id)
if err == nil { if err == nil {
if err := os.Rename(graph.ImageRoot(id), tmp); err != nil { if err := os.Rename(graph.imageRoot(id), tmp); err != nil {
// On err make tmp point to old dir and cleanup unused tmp dir // On err make tmp point to old dir and cleanup unused tmp dir
os.RemoveAll(tmp) os.RemoveAll(tmp)
tmp = graph.ImageRoot(id) tmp = graph.imageRoot(id)
} }
} else { } else {
// On err make tmp point to old dir for cleanup // On err make tmp point to old dir for cleanup
tmp = graph.ImageRoot(id) tmp = graph.imageRoot(id)
} }
// Remove rootfs data from the driver // Remove rootfs data from the driver
graph.driver.Remove(id) graph.driver.Remove(id)
@ -375,7 +382,7 @@ func (graph *Graph) Map() (map[string]*image.Image, error) {
// walkAll iterates over each image in the graph, and passes it to a handler. // walkAll iterates over each image in the graph, and passes it to a handler.
// The walking order is undetermined. // The walking order is undetermined.
func (graph *Graph) walkAll(handler func(*image.Image)) error { func (graph *Graph) walkAll(handler func(*image.Image)) error {
files, err := ioutil.ReadDir(graph.Root) files, err := ioutil.ReadDir(graph.root)
if err != nil { if err != nil {
return err return err
} }
@ -428,10 +435,125 @@ func (graph *Graph) Heads() (map[string]*image.Image, error) {
return heads, err return heads, err
} }
func (graph *Graph) ImageRoot(id string) string { func (graph *Graph) imageRoot(id string) string {
return filepath.Join(graph.Root, id) return filepath.Join(graph.root, id)
} }
func (graph *Graph) Driver() graphdriver.Driver { // storeImage stores file system layer data for the given image to the
return graph.driver // graph's storage driver. Image metadata is stored in a file
// at the specified root directory.
func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader, root string) (err error) {
// Store the layer. If layerData is not nil, unpack it into the new layer
if layerData != nil {
if img.Size, err = graph.driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil {
return err
}
}
if err := graph.saveSize(root, int(img.Size)); err != nil {
return err
}
f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
if err != nil {
return err
}
defer f.Close()
return json.NewEncoder(f).Encode(img)
}
// loadImage fetches the image with the given id from the graph.
func (graph *Graph) loadImage(id string) (*image.Image, error) {
root := graph.imageRoot(id)
// Open the JSON file to decode by streaming
jsonSource, err := os.Open(jsonPath(root))
if err != nil {
return nil, err
}
defer jsonSource.Close()
img := &image.Image{}
dec := json.NewDecoder(jsonSource)
// Decode the JSON data
if err := dec.Decode(img); err != nil {
return nil, err
}
if err := image.ValidateID(img.ID); err != nil {
return nil, err
}
if buf, err := ioutil.ReadFile(filepath.Join(root, "layersize")); err != nil {
if !os.IsNotExist(err) {
return nil, err
}
// If the layersize file does not exist then set the size to a negative number
// because a layer size of 0 (zero) is valid
img.Size = -1
} else {
// Using Atoi here instead would temporarily convert the size to a machine
// dependent integer type, which causes images larger than 2^31 bytes to
// display negative sizes on 32-bit machines:
size, err := strconv.ParseInt(string(buf), 10, 64)
if err != nil {
return nil, err
}
img.Size = int64(size)
}
return img, nil
}
// saveSize stores the `size` in the provided graph `img` directory `root`.
func (graph *Graph) saveSize(root string, size int) error {
if err := ioutil.WriteFile(filepath.Join(root, "layersize"), []byte(strconv.Itoa(size)), 0600); err != nil {
return fmt.Errorf("Error storing image size in %s/layersize: %s", root, err)
}
return nil
}
// SetDigest sets the digest for the image layer to the provided value.
func (graph *Graph) SetDigest(id string, dgst digest.Digest) error {
root := graph.imageRoot(id)
if err := ioutil.WriteFile(filepath.Join(root, "checksum"), []byte(dgst.String()), 0600); err != nil {
return fmt.Errorf("Error storing digest in %s/checksum: %s", root, err)
}
return nil
}
// GetDigest gets the digest for the provide image layer id.
func (graph *Graph) GetDigest(id string) (digest.Digest, error) {
root := graph.imageRoot(id)
cs, err := ioutil.ReadFile(filepath.Join(root, "checksum"))
if err != nil {
if os.IsNotExist(err) {
return "", ErrDigestNotSet
}
return "", err
}
return digest.ParseDigest(string(cs))
}
// RawJSON returns the JSON representation for an image as a byte array.
func (graph *Graph) RawJSON(id string) ([]byte, error) {
root := graph.imageRoot(id)
buf, err := ioutil.ReadFile(jsonPath(root))
if err != nil {
return nil, fmt.Errorf("Failed to read json for image %s: %s", id, err)
}
return buf, nil
}
func jsonPath(root string) string {
return filepath.Join(root, "json")
}
// TarLayer returns a tar archive of the image's filesystem layer.
func (graph *Graph) TarLayer(img *image.Image) (arch archive.Archive, err error) {
return graph.driver.Diff(img.ID, img.Parent)
} }

View File

@ -17,7 +17,7 @@ import (
func TestMount(t *testing.T) { func TestMount(t *testing.T) {
graph, driver := tempGraph(t) graph, driver := tempGraph(t)
defer os.RemoveAll(graph.Root) defer os.RemoveAll(graph.root)
defer driver.Cleanup() defer driver.Cleanup()
archive, err := fakeTar() archive, err := fakeTar()
@ -52,7 +52,7 @@ func TestInit(t *testing.T) {
graph, _ := tempGraph(t) graph, _ := tempGraph(t)
defer nukeGraph(graph) defer nukeGraph(graph)
// Root should exist // Root should exist
if _, err := os.Stat(graph.Root); err != nil { if _, err := os.Stat(graph.root); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Map() should be empty // Map() should be empty
@ -301,6 +301,6 @@ func tempGraph(t *testing.T) (*Graph, graphdriver.Driver) {
} }
func nukeGraph(graph *Graph) { func nukeGraph(graph *Graph) {
graph.Driver().Cleanup() graph.driver.Cleanup()
os.RemoveAll(graph.Root) os.RemoveAll(graph.root)
} }

View File

@ -1,6 +1,7 @@
package graph package graph
import ( import (
"fmt"
"strings" "strings"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
@ -8,6 +9,64 @@ import (
"github.com/docker/docker/utils" "github.com/docker/docker/utils"
) )
// WalkHistory calls the handler function for each image in the
// provided images lineage starting from immediate parent.
func (graph *Graph) WalkHistory(img *image.Image, handler func(image.Image) error) (err error) {
currentImg := img
for currentImg != nil {
if handler != nil {
if err := handler(*currentImg); err != nil {
return err
}
}
currentImg, err = graph.GetParent(currentImg)
if err != nil {
return fmt.Errorf("Error while getting parent image: %v", err)
}
}
return nil
}
// depth returns the number of parents for a
// current image
func (graph *Graph) depth(img *image.Image) (int, error) {
var (
count = 0
parent = img
err error
)
for parent != nil {
count++
parent, err = graph.GetParent(parent)
if err != nil {
return -1, err
}
}
return count, nil
}
// Set the max depth to the aufs default that most
// kernels are compiled with
// For more information see: http://sourceforge.net/p/aufs/aufs3-standalone/ci/aufs3.12/tree/config.mk
const MaxImageDepth = 127
// CheckDepth returns an error if the depth of an image, as returned
// by ImageDepth, is too large to support creating a container from it
// on this daemon.
func (graph *Graph) CheckDepth(img *image.Image) error {
// We add 2 layers to the depth because the container's rw and
// init layer add to the restriction
depth, err := graph.depth(img)
if err != nil {
return err
}
if depth+2 >= MaxImageDepth {
return fmt.Errorf("Cannot create container with more than %d parents", MaxImageDepth)
}
return nil
}
func (s *TagStore) History(name string) ([]*types.ImageHistory, error) { func (s *TagStore) History(name string) ([]*types.ImageHistory, error) {
foundImage, err := s.LookupImage(name) foundImage, err := s.LookupImage(name)
if err != nil { if err != nil {
@ -27,7 +86,7 @@ func (s *TagStore) History(name string) ([]*types.ImageHistory, error) {
history := []*types.ImageHistory{} history := []*types.ImageHistory{}
err = foundImage.WalkHistory(func(img *image.Image) error { err = s.graph.WalkHistory(foundImage, func(img image.Image) error {
history = append(history, &types.ImageHistory{ history = append(history, &types.ImageHistory{
ID: img.ID, ID: img.ID,
Created: img.Created.Unix(), Created: img.Created.Unix(),
@ -41,3 +100,19 @@ func (s *TagStore) History(name string) ([]*types.ImageHistory, error) {
return history, err return history, err
} }
func (graph *Graph) GetParent(img *image.Image) (*image.Image, error) {
if img.Parent == "" {
return nil, nil
}
return graph.Get(img.Parent)
}
func (graph *Graph) GetParentsSize(img *image.Image, size int64) int64 {
parentImage, err := graph.GetParent(img)
if err != nil || parentImage == nil {
return size
}
size += parentImage.Size
return graph.GetParentsSize(parentImage, size)
}

View File

@ -103,7 +103,7 @@ func (s *TagStore) Images(config *ImagesConfig) ([]*types.Image, error) {
newImage.ID = image.ID newImage.ID = image.ID
newImage.Created = int(image.Created.Unix()) newImage.Created = int(image.Created.Unix())
newImage.Size = int(image.Size) newImage.Size = int(image.Size)
newImage.VirtualSize = int(image.GetParentsSize(0) + image.Size) newImage.VirtualSize = int(s.graph.GetParentsSize(image, 0) + image.Size)
newImage.Labels = image.ContainerConfig.Labels newImage.Labels = image.ContainerConfig.Labels
if utils.DigestReference(ref) { if utils.DigestReference(ref) {
@ -140,7 +140,7 @@ func (s *TagStore) Images(config *ImagesConfig) ([]*types.Image, error) {
newImage.ID = image.ID newImage.ID = image.ID
newImage.Created = int(image.Created.Unix()) newImage.Created = int(image.Created.Unix())
newImage.Size = int(image.Size) newImage.Size = int(image.Size)
newImage.VirtualSize = int(image.GetParentsSize(0) + image.Size) newImage.VirtualSize = int(s.graph.GetParentsSize(image, 0) + image.Size)
newImage.Labels = image.ContainerConfig.Labels newImage.Labels = image.ContainerConfig.Labels
images = append(images, newImage) images = append(images, newImage)

View File

@ -3,14 +3,11 @@ package graph
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"io/ioutil"
"os" "os"
"testing" "testing"
"github.com/docker/distribution/digest" "github.com/docker/distribution/digest"
"github.com/docker/docker/image" "github.com/docker/docker/image"
"github.com/docker/docker/pkg/tarsum"
"github.com/docker/docker/registry" "github.com/docker/docker/registry"
"github.com/docker/docker/runconfig" "github.com/docker/docker/runconfig"
"github.com/docker/docker/utils" "github.com/docker/docker/utils"
@ -57,7 +54,7 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error
metadata = *layer.Config metadata = *layer.Config
} }
for ; layer != nil; layer, err = layer.GetParent() { for ; layer != nil; layer, err = s.graph.GetParent(layer) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -72,40 +69,34 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error
} }
} }
checksum, err := layer.GetCheckSum(s.graph.ImageRoot(layer.ID)) dgst, err := s.graph.GetDigest(layer.ID)
if err != nil { if err == ErrDigestNotSet {
return nil, fmt.Errorf("Error getting image checksum: %s", err) archive, err := s.graph.TarLayer(layer)
}
if tarsum.VersionLabelForChecksum(checksum) != tarsum.Version1.String() {
archive, err := layer.TarLayer()
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer archive.Close() defer archive.Close()
tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version1) dgst, err = digest.FromReader(archive)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
return nil, err
}
checksum = tarSum.Sum(nil)
// Save checksum value // Save checksum value
if err := layer.SaveCheckSum(s.graph.ImageRoot(layer.ID), checksum); err != nil { if err := s.graph.SetDigest(layer.ID, dgst); err != nil {
return nil, err return nil, err
} }
} else if err != nil {
return nil, fmt.Errorf("Error getting image checksum: %s", err)
} }
jsonData, err := layer.RawJson() jsonData, err := s.graph.RawJSON(layer.ID)
if err != nil { if err != nil {
return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err) return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)
} }
manifest.FSLayers = append(manifest.FSLayers, &registry.FSLayer{BlobSum: checksum}) manifest.FSLayers = append(manifest.FSLayers, &registry.FSLayer{BlobSum: dgst.String()})
layersSeen[layer.ID] = true layersSeen[layer.ID] = true
@ -141,10 +132,10 @@ func TestManifestTarsumCache(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if cs, err := img.GetCheckSum(store.graph.ImageRoot(testManifestImageID)); err != nil { if _, err := store.graph.GetDigest(testManifestImageID); err == nil {
t.Fatal(err)
} else if cs != "" {
t.Fatalf("Non-empty checksum file after register") t.Fatalf("Non-empty checksum file after register")
} else if err != ErrDigestNotSet {
t.Fatal(err)
} }
// Generate manifest // Generate manifest
@ -153,7 +144,7 @@ func TestManifestTarsumCache(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
manifestChecksum, err := img.GetCheckSum(store.graph.ImageRoot(testManifestImageID)) manifestChecksum, err := store.graph.GetDigest(testManifestImageID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -167,7 +158,7 @@ func TestManifestTarsumCache(t *testing.T) {
t.Fatalf("Unexpected number of layers, expecting 1: %d", len(manifest.FSLayers)) t.Fatalf("Unexpected number of layers, expecting 1: %d", len(manifest.FSLayers))
} }
if manifest.FSLayers[0].BlobSum != manifestChecksum { if manifest.FSLayers[0].BlobSum != manifestChecksum.String() {
t.Fatalf("Unexpected blob sum, expecting %q, got %q", manifestChecksum, manifest.FSLayers[0].BlobSum) t.Fatalf("Unexpected blob sum, expecting %q, got %q", manifestChecksum, manifest.FSLayers[0].BlobSum)
} }
@ -175,7 +166,7 @@ func TestManifestTarsumCache(t *testing.T) {
t.Fatalf("Unexpected number of layer history, expecting 1: %d", len(manifest.History)) t.Fatalf("Unexpected number of layer history, expecting 1: %d", len(manifest.History))
} }
v1compat, err := img.RawJson() v1compat, err := store.graph.RawJSON(img.ID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -207,10 +198,10 @@ func TestManifestDigestCheck(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if cs, err := img.GetCheckSum(store.graph.ImageRoot(testManifestImageID)); err != nil { if _, err := store.graph.GetDigest(testManifestImageID); err == nil {
t.Fatal(err)
} else if cs != "" {
t.Fatalf("Non-empty checksum file after register") t.Fatalf("Non-empty checksum file after register")
} else if err != ErrDigestNotSet {
t.Fatal(err)
} }
// Generate manifest // Generate manifest

View File

@ -5,9 +5,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"os" "os"
"path/filepath"
"sync" "sync"
"github.com/Sirupsen/logrus" "github.com/Sirupsen/logrus"
@ -57,7 +55,7 @@ func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string
tagsByImage[id] = append(tagsByImage[id], tag) tagsByImage[id] = append(tagsByImage[id], tag)
for img, err := s.graph.Get(id); img != nil; img, err = img.GetParent() { for img, err := s.graph.Get(id); img != nil; img, err = s.graph.GetParent(img) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -248,7 +246,7 @@ func (s *TagStore) pushRepository(r *registry.Session, out io.Writer,
func (s *TagStore) pushImage(r *registry.Session, out io.Writer, imgID, ep string, token []string, sf *streamformatter.StreamFormatter) (checksum string, err error) { func (s *TagStore) pushImage(r *registry.Session, out io.Writer, imgID, ep string, token []string, sf *streamformatter.StreamFormatter) (checksum string, err error) {
out = ioutils.NewWriteFlusher(out) out = ioutils.NewWriteFlusher(out)
jsonRaw, err := ioutil.ReadFile(filepath.Join(s.graph.Root, imgID, "json")) jsonRaw, err := s.graph.RawJSON(imgID)
if err != nil { if err != nil {
return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err)
} }
@ -349,7 +347,7 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o
layersSeen := make(map[string]bool) layersSeen := make(map[string]bool)
layers := []*image.Image{layer} layers := []*image.Image{layer}
for ; layer != nil; layer, err = layer.GetParent() { for ; layer != nil; layer, err = s.graph.GetParent(layer) {
if err != nil { if err != nil {
return err return err
} }
@ -372,23 +370,18 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o
return err return err
} }
} }
jsonData, err := layer.RawJson() jsonData, err := s.graph.RawJSON(layer.ID)
if err != nil { if err != nil {
return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err) return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err)
} }
checksum, err := layer.GetCheckSum(s.graph.ImageRoot(layer.ID))
if err != nil {
return fmt.Errorf("error getting image checksum: %s", err)
}
var exists bool var exists bool
if len(checksum) > 0 { dgst, err := s.graph.GetDigest(layer.ID)
dgst, err := digest.ParseDigest(checksum) if err != nil {
if err != nil { if err != ErrDigestNotSet {
return fmt.Errorf("Invalid checksum %s: %s", checksum, err) return fmt.Errorf("error getting image checksum: %s", err)
} }
} else {
// Call mount blob // Call mount blob
exists, err = r.HeadV2ImageBlob(endpoint, repoInfo.RemoteName, dgst, auth) exists, err = r.HeadV2ImageBlob(endpoint, repoInfo.RemoteName, dgst, auth)
if err != nil { if err != nil {
@ -397,19 +390,19 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o
} }
} }
if !exists { if !exists {
if cs, err := s.pushV2Image(r, layer, endpoint, repoInfo.RemoteName, sf, out, auth); err != nil { if pushDigest, err := s.pushV2Image(r, layer, endpoint, repoInfo.RemoteName, sf, out, auth); err != nil {
return err return err
} else if cs != checksum { } else if pushDigest != dgst {
// Cache new checksum // Cache new checksum
if err := layer.SaveCheckSum(s.graph.ImageRoot(layer.ID), cs); err != nil { if err := s.graph.SetDigest(layer.ID, pushDigest); err != nil {
return err return err
} }
checksum = cs dgst = pushDigest
} }
} else { } else {
out.Write(sf.FormatProgress(stringid.TruncateID(layer.ID), "Image already exists", nil)) out.Write(sf.FormatProgress(stringid.TruncateID(layer.ID), "Image already exists", nil))
} }
m.FSLayers[i] = &registry.FSLayer{BlobSum: checksum} m.FSLayers[i] = &registry.FSLayer{BlobSum: dgst.String()}
m.History[i] = &registry.ManifestHistory{V1Compatibility: string(jsonData)} m.History[i] = &registry.ManifestHistory{V1Compatibility: string(jsonData)}
} }
@ -449,14 +442,14 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o
} }
// PushV2Image pushes the image content to the v2 registry, first buffering the contents to disk // PushV2Image pushes the image content to the v2 registry, first buffering the contents to disk
func (s *TagStore) pushV2Image(r *registry.Session, img *image.Image, endpoint *registry.Endpoint, imageName string, sf *streamformatter.StreamFormatter, out io.Writer, auth *registry.RequestAuthorization) (string, error) { func (s *TagStore) pushV2Image(r *registry.Session, img *image.Image, endpoint *registry.Endpoint, imageName string, sf *streamformatter.StreamFormatter, out io.Writer, auth *registry.RequestAuthorization) (digest.Digest, error) {
out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Buffering to Disk", nil)) out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Buffering to Disk", nil))
image, err := s.graph.Get(img.ID) image, err := s.graph.Get(img.ID)
if err != nil { if err != nil {
return "", err return "", err
} }
arch, err := image.TarLayer() arch, err := s.graph.TarLayer(image)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -490,7 +483,7 @@ func (s *TagStore) pushV2Image(r *registry.Session, img *image.Image, endpoint *
return "", err return "", err
} }
out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Image successfully pushed", nil)) out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Image successfully pushed", nil))
return dgst.String(), nil return dgst, nil
} }
// FIXME: Allow to interrupt current push when new push of same image is done. // FIXME: Allow to interrupt current push when new push of same image is done.

View File

@ -14,7 +14,7 @@ func (s *TagStore) LookupRaw(name string) ([]byte, error) {
return nil, fmt.Errorf("No such image %s", name) return nil, fmt.Errorf("No such image %s", name)
} }
imageInspectRaw, err := image.RawJson() imageInspectRaw, err := s.graph.RawJSON(image.ID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -42,7 +42,7 @@ func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) {
Architecture: image.Architecture, Architecture: image.Architecture,
Os: image.OS, Os: image.OS,
Size: image.Size, Size: image.Size,
VirtualSize: image.GetParentsSize(0) + image.Size, VirtualSize: s.graph.GetParentsSize(image, 0) + image.Size,
} }
return imageInspect, nil return imageInspect, nil
@ -51,7 +51,7 @@ func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) {
// ImageTarLayer return the tarLayer of the image // ImageTarLayer return the tarLayer of the image
func (s *TagStore) ImageTarLayer(name string, dest io.Writer) error { func (s *TagStore) ImageTarLayer(name string, dest io.Writer) error {
if image, err := s.LookupImage(name); err == nil && image != nil { if image, err := s.LookupImage(name); err == nil && image != nil {
fs, err := image.TarLayer() fs, err := s.graph.TarLayer(image)
if err != nil { if err != nil {
return err return err
} }

View File

@ -1,11 +0,0 @@
package image
import (
"github.com/docker/docker/daemon/graphdriver"
)
type Graph interface {
Get(id string) (*Image, error)
ImageRoot(id string) string
Driver() graphdriver.Driver
}

View File

@ -3,22 +3,12 @@ package image
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp" "regexp"
"strconv"
"time" "time"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/runconfig" "github.com/docker/docker/runconfig"
) )
// Set the max depth to the aufs default that most
// kernels are compiled with
// For more information see: http://sourceforge.net/p/aufs/aufs3-standalone/ci/aufs3.12/tree/config.mk
const MaxImageDepth = 127
var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
type Image struct { type Image struct {
@ -34,225 +24,6 @@ type Image struct {
Architecture string `json:"architecture,omitempty"` Architecture string `json:"architecture,omitempty"`
OS string `json:"os,omitempty"` OS string `json:"os,omitempty"`
Size int64 Size int64
graph Graph
}
func LoadImage(root string) (*Image, error) {
// Open the JSON file to decode by streaming
jsonSource, err := os.Open(jsonPath(root))
if err != nil {
return nil, err
}
defer jsonSource.Close()
img := &Image{}
dec := json.NewDecoder(jsonSource)
// Decode the JSON data
if err := dec.Decode(img); err != nil {
return nil, err
}
if err := ValidateID(img.ID); err != nil {
return nil, err
}
if buf, err := ioutil.ReadFile(filepath.Join(root, "layersize")); err != nil {
if !os.IsNotExist(err) {
return nil, err
}
// If the layersize file does not exist then set the size to a negative number
// because a layer size of 0 (zero) is valid
img.Size = -1
} else {
// Using Atoi here instead would temporarily convert the size to a machine
// dependent integer type, which causes images larger than 2^31 bytes to
// display negative sizes on 32-bit machines:
size, err := strconv.ParseInt(string(buf), 10, 64)
if err != nil {
return nil, err
}
img.Size = int64(size)
}
return img, nil
}
// StoreImage stores file system layer data for the given image to the
// image's registered storage driver. Image metadata is stored in a file
// at the specified root directory.
func StoreImage(img *Image, layerData archive.ArchiveReader, root string) (err error) {
// Store the layer. If layerData is not nil, unpack it into the new layer
if layerData != nil {
if img.Size, err = img.graph.Driver().ApplyDiff(img.ID, img.Parent, layerData); err != nil {
return err
}
}
if err := img.SaveSize(root); err != nil {
return err
}
f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
if err != nil {
return err
}
defer f.Close()
return json.NewEncoder(f).Encode(img)
}
func (img *Image) SetGraph(graph Graph) {
img.graph = graph
}
// SaveSize stores the current `size` value of `img` in the directory `root`.
func (img *Image) SaveSize(root string) error {
if err := ioutil.WriteFile(filepath.Join(root, "layersize"), []byte(strconv.Itoa(int(img.Size))), 0600); err != nil {
return fmt.Errorf("Error storing image size in %s/layersize: %s", root, err)
}
return nil
}
func (img *Image) SaveCheckSum(root, checksum string) error {
if err := ioutil.WriteFile(filepath.Join(root, "checksum"), []byte(checksum), 0600); err != nil {
return fmt.Errorf("Error storing checksum in %s/checksum: %s", root, err)
}
return nil
}
func (img *Image) GetCheckSum(root string) (string, error) {
cs, err := ioutil.ReadFile(filepath.Join(root, "checksum"))
if err != nil {
if os.IsNotExist(err) {
return "", nil
}
return "", err
}
return string(cs), err
}
func jsonPath(root string) string {
return filepath.Join(root, "json")
}
func (img *Image) RawJson() ([]byte, error) {
root, err := img.root()
if err != nil {
return nil, fmt.Errorf("Failed to get root for image %s: %s", img.ID, err)
}
buf, err := ioutil.ReadFile(jsonPath(root))
if err != nil {
return nil, fmt.Errorf("Failed to read json for image %s: %s", img.ID, err)
}
return buf, nil
}
// TarLayer returns a tar archive of the image's filesystem layer.
func (img *Image) TarLayer() (arch archive.Archive, err error) {
if img.graph == nil {
return nil, fmt.Errorf("Can't load storage driver for unregistered image %s", img.ID)
}
driver := img.graph.Driver()
return driver.Diff(img.ID, img.Parent)
}
// Image includes convenience proxy functions to its graph
// These functions will return an error if the image is not registered
// (ie. if image.graph == nil)
func (img *Image) History() ([]*Image, error) {
var parents []*Image
if err := img.WalkHistory(
func(img *Image) error {
parents = append(parents, img)
return nil
},
); err != nil {
return nil, err
}
return parents, nil
}
func (img *Image) WalkHistory(handler func(*Image) error) (err error) {
currentImg := img
for currentImg != nil {
if handler != nil {
if err := handler(currentImg); err != nil {
return err
}
}
currentImg, err = currentImg.GetParent()
if err != nil {
return fmt.Errorf("Error while getting parent image: %v", err)
}
}
return nil
}
func (img *Image) GetParent() (*Image, error) {
if img.Parent == "" {
return nil, nil
}
if img.graph == nil {
return nil, fmt.Errorf("Can't lookup parent of unregistered image")
}
return img.graph.Get(img.Parent)
}
func (img *Image) root() (string, error) {
if img.graph == nil {
return "", fmt.Errorf("Can't lookup root of unregistered image")
}
return img.graph.ImageRoot(img.ID), nil
}
func (img *Image) GetParentsSize(size int64) int64 {
parentImage, err := img.GetParent()
if err != nil || parentImage == nil {
return size
}
size += parentImage.Size
return parentImage.GetParentsSize(size)
}
// Depth returns the number of parents for a
// current image
func (img *Image) Depth() (int, error) {
var (
count = 0
parent = img
err error
)
for parent != nil {
count++
parent, err = parent.GetParent()
if err != nil {
return -1, err
}
}
return count, nil
}
// CheckDepth returns an error if the depth of an image, as returned
// by ImageDepth, is too large to support creating a container from it
// on this daemon.
func (img *Image) CheckDepth() error {
// We add 2 layers to the depth because the container's rw and
// init layer add to the restriction
depth, err := img.Depth()
if err != nil {
return err
}
if depth+2 >= MaxImageDepth {
return fmt.Errorf("Cannot create container with more than %d parents", MaxImageDepth)
}
return nil
} }
// Build an Image object from raw json data // Build an Image object from raw json data