docker
/
docs
mirror of https://github.com/docker/docs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove graph package 

The graph package is no longer used for tag, image, or layer storage.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi 2015-11-18 14:21:23 -08:00 committed by Aaron Lehmann
parent 4352da7803
commit ed4d236e04
28 changed files with 0 additions and 5237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
graph/export.go
View File

@ -1,180 +0,0 @@
package graph
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/registry"
)
// ImageExport exports list of images to a output stream specified in the
// config. The exported images are archived into a tar when written to the
// output stream. All images with the given tag and all versions containing the
// same tag are exported. names is the set of tags to export, and outStream
// is the writer which the images are written to.
func (s *TagStore) ImageExport(names []string, outStream io.Writer) error {
// get image json
tempdir, err := ioutil.TempDir("", "docker-export-")
if err != nil {
return err
}
defer os.RemoveAll(tempdir)
rootRepoMap := map[string]repository{}
addKey := func(name string, tag string, id string) {
logrus.Debugf("add key [%s:%s]", name, tag)
if repo, ok := rootRepoMap[name]; !ok {
rootRepoMap[name] = repository{tag: id}
} else {
repo[tag] = id
}
}
for _, name := range names {
name = registry.NormalizeLocalName(name)
logrus.Debugf("Serializing %s", name)
rootRepo := s.Repositories[name]
if rootRepo != nil {
// this is a base repo name, like 'busybox'
for tag, id := range rootRepo {
addKey(name, tag, id)
if err := s.exportImage(id, tempdir); err != nil {
return err
}
}
} else {
img, err := s.LookupImage(name)
if err != nil {
return err
}
if img != nil {
// This is a named image like 'busybox:latest'
repoName, repoTag := parsers.ParseRepositoryTag(name)
// Skip digests on save
if _, err := digest.ParseDigest(repoTag); err == nil {
repoTag = ""
}
// check this length, because a lookup of a truncated has will not have a tag
// and will not need to be added to this map
if len(repoTag) > 0 {
addKey(repoName, repoTag, img.ID)
}
if err := s.exportImage(img.ID, tempdir); err != nil {
return err
}
} else {
// this must be an ID that didn't get looked up just right?
if err := s.exportImage(name, tempdir); err != nil {
return err
}
}
}
logrus.Debugf("End Serializing %s", name)
}
// write repositories, if there is something to write
if len(rootRepoMap) > 0 {
f, err := os.OpenFile(filepath.Join(tempdir, "repositories"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
f.Close()
return err
}
if err := json.NewEncoder(f).Encode(rootRepoMap); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
if err := os.Chtimes(filepath.Join(tempdir, "repositories"), time.Unix(0, 0), time.Unix(0, 0)); err != nil {
return err
}
} else {
logrus.Debugf("There were no repositories to write")
}
fs, err := archive.Tar(tempdir, archive.Uncompressed)
if err != nil {
return err
}
defer fs.Close()
if _, err := io.Copy(outStream, fs); err != nil {
return err
}
logrus.Debugf("End export image")
return nil
}
func (s *TagStore) exportImage(name, tempdir string) error {
for n := name; n != ""; {
img, err := s.LookupImage(n)
if err != nil || img == nil {
return fmt.Errorf("No such image %s", n)
}
// temporary directory
tmpImageDir := filepath.Join(tempdir, n)
if err := os.Mkdir(tmpImageDir, os.FileMode(0755)); err != nil {
if os.IsExist(err) {
return nil
}
return err
}
var version = "1.0"
var versionBuf = []byte(version)
if err := ioutil.WriteFile(filepath.Join(tmpImageDir, "VERSION"), versionBuf, os.FileMode(0644)); err != nil {
return err
}
imageInspectRaw, err := json.Marshal(img)
if err != nil {
return err
}
// serialize json
json, err := os.Create(filepath.Join(tmpImageDir, "json"))
if err != nil {
return err
}
written, err := json.Write(imageInspectRaw)
if err != nil {
return err
}
if written != len(imageInspectRaw) {
logrus.Warnf("%d byes should have been written instead %d have been written", written, len(imageInspectRaw))
}
// serialize filesystem
fsTar, err := os.Create(filepath.Join(tmpImageDir, "layer.tar"))
if err != nil {
return err
}
if err := s.imageTarLayer(n, fsTar); err != nil {
return err
}
for _, fname := range []string{"", "VERSION", "json", "layer.tar"} {
if err := os.Chtimes(filepath.Join(tmpImageDir, fname), img.Created, img.Created); err != nil {
return err
}
}
// try again with parent
n = img.Parent
}
return nil
}

38
graph/fixtures/validate_manifest/bad_manifest
View File

@ -1,38 +0,0 @@
{
"schemaVersion": 2,
"name": "library/hello-world",
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
},
{
"blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
}
],
"history": [
{
"v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
},
{
"v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
}
],
"signatures": [
{
"header": {
"jwk": {
"crv": "P-256",
"kid": "OIH7:HQFS:44FK:45VB:3B53:OIAG:TPL4:ATF5:6PNE:MGHN:NHQX:2GE4",
"kty": "EC",
"x": "Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ",
"y": "zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8"
},
"alg": "ES256"
},
"signature": "Y6xaFz9Sy-OtcnKQS1Ilq3Dh8cu4h3nBTJCpOTF1XF7vKtcxxA_xMP8-SgDo869SJ3VsvgPL9-Xn-OoYG2rb1A",
"protected": "eyJmb3JtYXRMZW5ndGgiOjMxOTcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wOS0xMVQwNDoxMzo0OFoifQ"
}
]
}

46
graph/fixtures/validate_manifest/extra_data_manifest
View File

@ -1,46 +0,0 @@
{
"schemaVersion": 1,
"name": "library/hello-world",
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
},
{
"blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
}
],
"history": [
{
"v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
},
{
"v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
}
],
"fsLayers": [
{
"blobSum": "sha256:ffff95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
},
{
"blobSum": "sha256:ffff658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
}
],
"signatures": [
{
"header": {
"jwk": {
"crv": "P-256",
"kid": "OIH7:HQFS:44FK:45VB:3B53:OIAG:TPL4:ATF5:6PNE:MGHN:NHQX:2GE4",
"kty": "EC",
"x": "Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ",
"y": "zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8"
},
"alg": "ES256"
},
"signature": "Y6xaFz9Sy-OtcnKQS1Ilq3Dh8cu4h3nBTJCpOTF1XF7vKtcxxA_xMP8-SgDo869SJ3VsvgPL9-Xn-OoYG2rb1A",
"protected": "eyJmb3JtYXRMZW5ndGgiOjMxOTcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wOS0xMVQwNDoxMzo0OFoifQ"
}
]
}

38
graph/fixtures/validate_manifest/good_manifest
View File

@ -1,38 +0,0 @@
{
"schemaVersion": 1,
"name": "library/hello-world",
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
},
{
"blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
}
],
"history": [
{
"v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
},
{
"v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
}
],
"signatures": [
{
"header": {
"jwk": {
"crv": "P-256",
"kid": "OIH7:HQFS:44FK:45VB:3B53:OIAG:TPL4:ATF5:6PNE:MGHN:NHQX:2GE4",
"kty": "EC",
"x": "Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ",
"y": "zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8"
},
"alg": "ES256"
},
"signature": "Y6xaFz9Sy-OtcnKQS1Ilq3Dh8cu4h3nBTJCpOTF1XF7vKtcxxA_xMP8-SgDo869SJ3VsvgPL9-Xn-OoYG2rb1A",
"protected": "eyJmb3JtYXRMZW5ndGgiOjMxOTcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wOS0xMVQwNDoxMzo0OFoifQ"
}
]
}

22
graph/fixtures/validate_manifest/no_signature_manifest
View File

@ -1,22 +0,0 @@
{
"schemaVersion": 1,
"name": "library/hello-world",
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
},
{
"blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
}
],
"history": [
{
"v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
},
{
"v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
}
]
}

813
graph/graph.go
View File

@ -1,813 +0,0 @@
package graph
import (
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/dockerversion"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/locker"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/truncindex"
"github.com/docker/docker/runconfig"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
// v1Descriptor is a non-content-addressable image descriptor
type v1Descriptor struct {
img *image.Image
}
// ID returns the image ID specified in the image structure.
func (img v1Descriptor) ID() string {
return img.img.ID
}
// Parent returns the parent ID specified in the image structure.
func (img v1Descriptor) Parent() string {
return img.img.Parent
}
// MarshalConfig renders the image structure into JSON.
func (img v1Descriptor) MarshalConfig() ([]byte, error) {
return json.Marshal(img.img)
}
// The type is used to protect pulling or building related image
// layers from deleteing when filtered by dangling=true
// The key of layers is the images ID which is pulling or building
// The value of layers is a slice which hold layer IDs referenced to
// pulling or building images
type retainedLayers struct {
layerHolders map[string]map[string]struct{} // map[layerID]map[sessionID]
sync.Mutex
}
func (r *retainedLayers) Add(sessionID string, layerIDs []string) {
r.Lock()
defer r.Unlock()
for _, layerID := range layerIDs {
if r.layerHolders[layerID] == nil {
r.layerHolders[layerID] = map[string]struct{}{}
}
r.layerHolders[layerID][sessionID] = struct{}{}
}
}
func (r *retainedLayers) Delete(sessionID string, layerIDs []string) {
r.Lock()
defer r.Unlock()
for _, layerID := range layerIDs {
holders, ok := r.layerHolders[layerID]
if !ok {
continue
}
delete(holders, sessionID)
if len(holders) == 0 {
delete(r.layerHolders, layerID) // Delete any empty reference set.
}
}
}
func (r *retainedLayers) Exists(layerID string) bool {
r.Lock()
_, exists := r.layerHolders[layerID]
r.Unlock()
return exists
}
// A Graph is a store for versioned filesystem images and the relationship between them.
type Graph struct {
root string
idIndex *truncindex.TruncIndex
driver graphdriver.Driver
imagesMutex sync.Mutex
imageMutex locker.Locker // protect images in driver.
retained *retainedLayers
tarSplitDisabled bool
uidMaps []idtools.IDMap
gidMaps []idtools.IDMap
// access to parentRefs must be protected with imageMutex locking the image id
// on the key of the map (e.g. imageMutex.Lock(img.ID), parentRefs[img.ID]...)
parentRefs map[string]int
}
// file names for ./graph/<ID>/
const (
jsonFileName = "json"
layersizeFileName = "layersize"
digestFileName = "checksum"
tarDataFileName = "tar-data.json.gz"
v1CompatibilityFileName = "v1Compatibility"
parentFileName = "parent"
)
var (
// errDigestNotSet is used when request the digest for a layer
// but the layer has no digest value or content to compute the
// the digest.
errDigestNotSet = errors.New("digest is not set for layer")
)
// NewGraph instantiates a new graph at the given root path in the filesystem.
// `root` will be created if it doesn't exist.
func NewGraph(root string, driver graphdriver.Driver, uidMaps, gidMaps []idtools.IDMap) (*Graph, error) {
abspath, err := filepath.Abs(root)
if err != nil {
return nil, err
}
rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
if err != nil {
return nil, err
}
// Create the root directory if it doesn't exists
if err := idtools.MkdirAllAs(root, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {
return nil, err
}
graph := &Graph{
root: abspath,
idIndex: truncindex.NewTruncIndex([]string{}),
driver: driver,
retained: &retainedLayers{layerHolders: make(map[string]map[string]struct{})},
uidMaps: uidMaps,
gidMaps: gidMaps,
parentRefs: make(map[string]int),
}
// Windows does not currently support tarsplit functionality.
if runtime.GOOS == "windows" {
graph.tarSplitDisabled = true
}
if err := graph.restore(); err != nil {
return nil, err
}
return graph, nil
}
// IsHeld returns whether the given layerID is being used by an ongoing pull or build.
func (graph *Graph) IsHeld(layerID string) bool {
return graph.retained.Exists(layerID)
}
func (graph *Graph) restore() error {
dir, err := ioutil.ReadDir(graph.root)
if err != nil {
return err
}
var ids = []string{}
for _, v := range dir {
id := v.Name()
if graph.driver.Exists(id) {
img, err := graph.loadImage(id)
if err != nil {
logrus.Warnf("ignoring image %s, it could not be restored: %v", id, err)
continue
}
graph.imageMutex.Lock(img.Parent)
graph.parentRefs[img.Parent]++
graph.imageMutex.Unlock(img.Parent)
ids = append(ids, id)
}
}
graph.idIndex = truncindex.NewTruncIndex(ids)
logrus.Debugf("Restored %d elements", len(ids))
return nil
}
// IsNotExist detects whether an image exists by parsing the incoming error
// message.
func (graph *Graph) IsNotExist(err error, id string) bool {
// FIXME: Implement error subclass instead of looking at the error text
// Note: This is the way golang implements os.IsNotExists on Plan9
return err != nil && (strings.Contains(strings.ToLower(err.Error()), "does not exist") || strings.Contains(strings.ToLower(err.Error()), "no such")) && strings.Contains(err.Error(), id)
}
// Exists returns true if an image is registered at the given id.
// If the image doesn't exist or if an error is encountered, false is returned.
func (graph *Graph) Exists(id string) bool {
if _, err := graph.Get(id); err != nil {
return false
}
return true
}
// Get returns the image with the given id, or an error if the image doesn't exist.
func (graph *Graph) Get(name string) (*image.Image, error) {
id, err := graph.idIndex.Get(name)
if err != nil {
if err == truncindex.ErrNotExist {
return nil, fmt.Errorf("image %s does not exist", name)
}
return nil, err
}
img, err := graph.loadImage(id)
if err != nil {
return nil, err
}
if img.ID != id {
return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID)
}
if img.Size < 0 {
size, err := graph.driver.DiffSize(img.ID, img.Parent)
if err != nil {
return nil, fmt.Errorf("unable to calculate size of image id %q: %s", img.ID, err)
}
img.Size = size
if err := graph.saveSize(graph.imageRoot(id), img.Size); err != nil {
return nil, err
}
}
return img, nil
}
// Create creates a new image and registers it in the graph.
func (graph *Graph) Create(layerData io.Reader, containerID, containerImage, comment, author string, containerConfig, config *runconfig.Config) (*image.Image, error) {
img := &image.Image{
ID: stringid.GenerateRandomID(),
Comment: comment,
Created: time.Now().UTC(),
DockerVersion: dockerversion.Version,
Author: author,
Config: config,
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
}
if containerID != "" {
img.Parent = containerImage
img.Container = containerID
img.ContainerConfig = *containerConfig
}
if err := graph.Register(v1Descriptor{img}, layerData); err != nil {
return nil, err
}
return img, nil
}
// Register imports a pre-existing image into the graph.
// Returns nil if the image is already registered.
func (graph *Graph) Register(im image.Descriptor, layerData io.Reader) (err error) {
imgID := im.ID()
if err := image.ValidateID(imgID); err != nil {
return err
}
// this is needed cause pull_v2 attemptIDReuse could deadlock
graph.imagesMutex.Lock()
defer graph.imagesMutex.Unlock()
// We need this entire operation to be atomic within the engine. Note that
// this doesn't mean Register is fully safe yet.
graph.imageMutex.Lock(imgID)
defer graph.imageMutex.Unlock(imgID)
return graph.register(im, layerData)
}
func (graph *Graph) register(im image.Descriptor, layerData io.Reader) (err error) {
imgID := im.ID()
// Skip register if image is already registered
if graph.Exists(imgID) {
return nil
}
// The returned `error` must be named in this function's signature so that
// `err` is not shadowed in this deferred cleanup.
defer func() {
// If any error occurs, remove the new dir from the driver.
// Don't check for errors since the dir might not have been created.
if err != nil {
graph.driver.Remove(imgID)
}
}()
// Ensure that the image root does not exist on the filesystem
// when it is not registered in the graph.
// This is common when you switch from one graph driver to another
if err := os.RemoveAll(graph.imageRoot(imgID)); err != nil && !os.IsNotExist(err) {
return err
}
// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
// (the graph is the source of truth).
// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
// (FIXME: make that mandatory for drivers).
graph.driver.Remove(imgID)
tmp, err := graph.mktemp()
if err != nil {
return err
}
defer os.RemoveAll(tmp)
parent := im.Parent()
// Create root filesystem in the driver
if err := createRootFilesystemInDriver(graph, imgID, parent); err != nil {
return err
}
// Apply the diff/layer
config, err := im.MarshalConfig()
if err != nil {
return err
}
if err := graph.storeImage(imgID, parent, config, layerData, tmp); err != nil {
return err
}
// Commit
if err := os.Rename(tmp, graph.imageRoot(imgID)); err != nil {
return err
}
graph.idIndex.Add(imgID)
graph.imageMutex.Lock(parent)
graph.parentRefs[parent]++
graph.imageMutex.Unlock(parent)
return nil
}
func createRootFilesystemInDriver(graph *Graph, id, parent string) error {
if err := graph.driver.Create(id, parent, ""); err != nil {
return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, id, err)
}
return nil
}
// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
// The archive is stored on disk and will be automatically deleted as soon as has been read.
// If output is not nil, a human-readable progress bar will be written to it.
func (graph *Graph) tempLayerArchive(id string, sf *streamformatter.StreamFormatter, output io.Writer) (*archive.TempArchive, error) {
image, err := graph.Get(id)
if err != nil {
return nil, err
}
tmp, err := graph.mktemp()
if err != nil {
return nil, err
}
defer os.RemoveAll(tmp)
a, err := graph.tarLayer(image)
if err != nil {
return nil, err
}
progressReader := progressreader.New(progressreader.Config{
In: a,
Out: output,
Formatter: sf,
Size: 0,
NewLines: false,
ID: stringid.TruncateID(id),
Action: "Buffering to disk",
})
defer progressReader.Close()
return archive.NewTempArchive(progressReader, tmp)
}
// mktemp creates a temporary sub-directory inside the graph's filesystem.
func (graph *Graph) mktemp() (string, error) {
dir := filepath.Join(graph.root, "_tmp", stringid.GenerateNonCryptoID())
rootUID, rootGID, err := idtools.GetRootUIDGID(graph.uidMaps, graph.gidMaps)
if err != nil {
return "", err
}
if err := idtools.MkdirAllAs(dir, 0700, rootUID, rootGID); err != nil {
return "", err
}
return dir, nil
}
// Delete atomically removes an image from the graph.
func (graph *Graph) Delete(name string) error {
id, err := graph.idIndex.Get(name)
if err != nil {
return err
}
img, err := graph.Get(id)
if err != nil {
return err
}
graph.idIndex.Delete(id)
tmp, err := graph.mktemp()
if err != nil {
tmp = graph.imageRoot(id)
} else {
if err := os.Rename(graph.imageRoot(id), tmp); err != nil {
// On err make tmp point to old dir and cleanup unused tmp dir
os.RemoveAll(tmp)
tmp = graph.imageRoot(id)
}
}
// Remove rootfs data from the driver
graph.driver.Remove(id)
graph.imageMutex.Lock(img.Parent)
graph.parentRefs[img.Parent]--
if graph.parentRefs[img.Parent] == 0 {
delete(graph.parentRefs, img.Parent)
}
graph.imageMutex.Unlock(img.Parent)
// Remove the trashed image directory
return os.RemoveAll(tmp)
}
// Map returns a list of all images in the graph, addressable by ID.
func (graph *Graph) Map() map[string]*image.Image {
images := make(map[string]*image.Image)
graph.walkAll(func(image *image.Image) {
images[image.ID] = image
})
return images
}
// walkAll iterates over each image in the graph, and passes it to a handler.
// The walking order is undetermined.
func (graph *Graph) walkAll(handler func(*image.Image)) {
graph.idIndex.Iterate(func(id string) {
img, err := graph.Get(id)
if err != nil {
return
}
if handler != nil {
handler(img)
}
})
}
// ByParent returns a lookup table of images by their parent.
// If an image of key ID has 3 children images, then the value for key ID
// will be a list of 3 images.
// If an image has no children, it will not have an entry in the table.
func (graph *Graph) ByParent() map[string][]*image.Image {
byParent := make(map[string][]*image.Image)
graph.walkAll(func(img *image.Image) {
parent, err := graph.Get(img.Parent)
if err != nil {
return
}
byParent[parent.ID] = append(byParent[parent.ID], img)
})
return byParent
}
// HasChildren returns whether the given image has any child images.
func (graph *Graph) HasChildren(imgID string) bool {
graph.imageMutex.Lock(imgID)
count := graph.parentRefs[imgID]
graph.imageMutex.Unlock(imgID)
return count > 0
}
// Retain keeps the images and layers that are in the pulling chain so that
// they are not deleted. If not retained, they may be deleted by rmi.
func (graph *Graph) Retain(sessionID string, layerIDs ...string) {
graph.retained.Add(sessionID, layerIDs)
}
// Release removes the referenced image ID from the provided set of layers.
func (graph *Graph) Release(sessionID string, layerIDs ...string) {
graph.retained.Delete(sessionID, layerIDs)
}
// heads returns all heads in the graph, keyed by id.
// A head is an image which is not the parent of another image in the graph.
func (graph *Graph) heads() map[string]*image.Image {
heads := make(map[string]*image.Image)
graph.walkAll(func(image *image.Image) {
// if it has no children, then it's not a parent, so it's an head
if !graph.HasChildren(image.ID) {
heads[image.ID] = image
}
})
return heads
}
// tarLayer returns a tar archive of the image's filesystem layer.
func (graph *Graph) tarLayer(img *image.Image) (arch io.ReadCloser, err error) {
rdr, err := graph.assembleTarLayer(img)
if err != nil {
logrus.Debugf("[graph] tarLayer with traditional differ: %s", img.ID)
return graph.driver.Diff(img.ID, img.Parent)
}
return rdr, nil
}
func (graph *Graph) imageRoot(id string) string {
return filepath.Join(graph.root, id)
}
// loadImage fetches the image with the given id from the graph.
func (graph *Graph) loadImage(id string) (*image.Image, error) {
root := graph.imageRoot(id)
// Open the JSON file to decode by streaming
jsonSource, err := os.Open(jsonPath(root))
if err != nil {
return nil, err
}
defer jsonSource.Close()
img := &image.Image{}
dec := json.NewDecoder(jsonSource)
// Decode the JSON data
if err := dec.Decode(img); err != nil {
return nil, err
}
if img.ID == "" {
img.ID = id
}
if img.Parent == "" && img.ParentID != "" && img.ParentID.Validate() == nil {
img.Parent = img.ParentID.Hex()
}
// compatibilityID for parent
parent, err := ioutil.ReadFile(filepath.Join(root, parentFileName))
if err == nil && len(parent) > 0 {
img.Parent = string(parent)
}
if err := image.ValidateID(img.ID); err != nil {
return nil, err
}
if buf, err := ioutil.ReadFile(filepath.Join(root, layersizeFileName)); err != nil {
if !os.IsNotExist(err) {
return nil, err
}
// If the layersize file does not exist then set the size to a negative number
// because a layer size of 0 (zero) is valid
img.Size = -1
} else {
// Using Atoi here instead would temporarily convert the size to a machine
// dependent integer type, which causes images larger than 2^31 bytes to
// display negative sizes on 32-bit machines:
size, err := strconv.ParseInt(string(buf), 10, 64)
if err != nil {
return nil, err
}
img.Size = int64(size)
}
return img, nil
}
// saveSize stores the `size` in the provided graph `img` directory `root`.
func (graph *Graph) saveSize(root string, size int64) error {
if err := ioutil.WriteFile(filepath.Join(root, layersizeFileName), []byte(strconv.FormatInt(size, 10)), 0600); err != nil {
return fmt.Errorf("Error storing image size in %s/%s: %s", root, layersizeFileName, err)
}
return nil
}
// setLayerDigestWithLock sets the digest for the image layer to the provided value.
func (graph *Graph) setLayerDigestWithLock(id string, dgst digest.Digest) error {
graph.imageMutex.Lock(id)
defer graph.imageMutex.Unlock(id)
return graph.setLayerDigest(id, dgst)
}
func (graph *Graph) setLayerDigest(id string, dgst digest.Digest) error {
root := graph.imageRoot(id)
if err := ioutil.WriteFile(filepath.Join(root, digestFileName), []byte(dgst.String()), 0600); err != nil {
return fmt.Errorf("Error storing digest in %s/%s: %s", root, digestFileName, err)
}
return nil
}
// getLayerDigestWithLock gets the digest for the provide image layer id.
func (graph *Graph) getLayerDigestWithLock(id string) (digest.Digest, error) {
graph.imageMutex.Lock(id)
defer graph.imageMutex.Unlock(id)
return graph.getLayerDigest(id)
}
func (graph *Graph) getLayerDigest(id string) (digest.Digest, error) {
root := graph.imageRoot(id)
cs, err := ioutil.ReadFile(filepath.Join(root, digestFileName))
if err != nil {
if os.IsNotExist(err) {
return "", errDigestNotSet
}
return "", err
}
return digest.ParseDigest(string(cs))
}
// setV1CompatibilityConfig stores the v1Compatibility JSON data associated
// with the image in the manifest to the disk
func (graph *Graph) setV1CompatibilityConfig(id string, data []byte) error {
root := graph.imageRoot(id)
return ioutil.WriteFile(filepath.Join(root, v1CompatibilityFileName), data, 0600)
}
// getV1CompatibilityConfig reads the v1Compatibility JSON data for the image
// from the disk
func (graph *Graph) getV1CompatibilityConfig(id string) ([]byte, error) {
root := graph.imageRoot(id)
return ioutil.ReadFile(filepath.Join(root, v1CompatibilityFileName))
}
// generateV1CompatibilityChain makes sure v1Compatibility JSON data exists
// for the image. If it doesn't it generates and stores it for the image and
// all of it's parents based on the image config JSON.
func (graph *Graph) generateV1CompatibilityChain(id string) ([]byte, error) {
graph.imageMutex.Lock(id)
defer graph.imageMutex.Unlock(id)
if v1config, err := graph.getV1CompatibilityConfig(id); err == nil {
return v1config, nil
}
// generate new, store it to disk
img, err := graph.Get(id)
if err != nil {
return nil, err
}
digestPrefix := string(digest.Canonical) + ":"
img.ID = strings.TrimPrefix(img.ID, digestPrefix)
if img.Parent != "" {
parentConfig, err := graph.generateV1CompatibilityChain(img.Parent)
if err != nil {
return nil, err
}
var parent struct{ ID string }
err = json.Unmarshal(parentConfig, &parent)
if err != nil {
return nil, err
}
img.Parent = parent.ID
}
json, err := json.Marshal(img)
if err != nil {
return nil, err
}
if err := graph.setV1CompatibilityConfig(id, json); err != nil {
return nil, err
}
return json, nil
}
func jsonPath(root string) string {
return filepath.Join(root, jsonFileName)
}
// storeImage stores file system layer data for the given image to the
// graph's storage driver. Image metadata is stored in a file
// at the specified root directory.
func (graph *Graph) storeImage(id, parent string, config []byte, layerData io.Reader, root string) (err error) {
var size int64
// Store the layer. If layerData is not nil, unpack it into the new layer
if layerData != nil {
if size, err = graph.disassembleAndApplyTarLayer(id, parent, layerData, root); err != nil {
return err
}
}
if err := graph.saveSize(root, size); err != nil {
return err
}
if err := ioutil.WriteFile(jsonPath(root), config, 0600); err != nil {
return err
}
// If image is pointing to a parent via CompatibilityID write the reference to disk
img, err := image.NewImgJSON(config)
if err != nil {
return err
}
if (img.ParentID.Validate() == nil && parent != img.ParentID.Hex()) || (allowBaseParentImage && img.ParentID == "" && parent != "") {
// save compatibilityID parent if it doesn't match parentID
// on windows always save a parent file pointing to the base layer
if err := ioutil.WriteFile(filepath.Join(root, parentFileName), []byte(parent), 0600); err != nil {
return err
}
}
return nil
}
func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData io.Reader, root string) (size int64, err error) {
var ar io.Reader
if graph.tarSplitDisabled {
ar = layerData
} else {
// this is saving the tar-split metadata
mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
if err != nil {
return 0, err
}
mfz := gzip.NewWriter(mf)
metaPacker := storage.NewJSONPacker(mfz)
defer mf.Close()
defer mfz.Close()
inflatedLayerData, err := archive.DecompressStream(layerData)
if err != nil {
return 0, err
}
// we're passing nil here for the file putter, because the ApplyDiff will
// handle the extraction of the archive
rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil)
if err != nil {
return 0, err
}
ar = archive.Reader(rdr)
}
if size, err = graph.driver.ApplyDiff(id, parent, ar); err != nil {
return 0, err
}
return
}
func (graph *Graph) assembleTarLayer(img *image.Image) (io.ReadCloser, error) {
root := graph.imageRoot(img.ID)
mFileName := filepath.Join(root, tarDataFileName)
mf, err := os.Open(mFileName)
if err != nil {
if !os.IsNotExist(err) {
logrus.Errorf("failed to open %q: %s", mFileName, err)
}
return nil, err
}
pR, pW := io.Pipe()
// this will need to be in a goroutine, as we are returning the stream of a
// tar archive, but can not close the metadata reader early (when this
// function returns)...
go func() {
defer mf.Close()
// let's reassemble!
logrus.Debugf("[graph] TarLayer with reassembly: %s", img.ID)
mfz, err := gzip.NewReader(mf)
if err != nil {
pW.CloseWithError(fmt.Errorf("[graph] error with %s: %s", mFileName, err))
return
}
defer mfz.Close()
// get our relative path to the container
fsLayer, err := graph.driver.Get(img.ID, "")
if err != nil {
pW.CloseWithError(err)
return
}
defer graph.driver.Put(img.ID)
metaUnpacker := storage.NewJSONUnpacker(mfz)
fileGetter := storage.NewPathFileGetter(fsLayer)
logrus.Debugf("[graph] %s is at %q", img.ID, fsLayer)
ots := asm.NewOutputTarStream(fileGetter, metaUnpacker)
defer ots.Close()
if _, err := io.Copy(pW, ots); err != nil {
pW.CloseWithError(err)
return
}
pW.Close()
}()
return pR, nil
}

308
graph/graph_test.go
View File

@ -1,308 +0,0 @@
package graph
import (
"errors"
"io"
"io/ioutil"
"os"
"path"
"testing"
"time"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/dockerversion"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/stringid"
)
func TestMount(t *testing.T) {
graph, driver := tempGraph(t)
defer os.RemoveAll(graph.root)
defer driver.Cleanup()
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
image, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
if err != nil {
t.Fatal(err)
}
tmp, err := ioutil.TempDir("", "docker-test-graph-mount-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
rootfs := path.Join(tmp, "rootfs")
if err := os.MkdirAll(rootfs, 0700); err != nil {
t.Fatal(err)
}
rw := path.Join(tmp, "rw")
if err := os.MkdirAll(rw, 0700); err != nil {
t.Fatal(err)
}
if _, err := driver.Get(image.ID, ""); err != nil {
t.Fatal(err)
}
}
func TestInit(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
// Root should exist
if _, err := os.Stat(graph.root); err != nil {
t.Fatal(err)
}
// Map() should be empty
l := graph.Map()
if len(l) != 0 {
t.Fatalf("len(Map()) should return %d, not %d", 0, len(l))
}
}
// Test that Register can be interrupted cleanly without side effects
func TestInterruptedRegister(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
badArchive, w := io.Pipe() // Use a pipe reader as a fake archive which never yields data
image := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "testing",
Created: time.Now(),
}
w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling)
graph.Register(v1Descriptor{image}, badArchive)
if _, err := graph.Get(image.ID); err == nil {
t.Fatal("Image should not exist after Register is interrupted")
}
// Registering the same image again should succeed if the first register was interrupted
goodArchive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
if err := graph.Register(v1Descriptor{image}, goodArchive); err != nil {
t.Fatal(err)
}
}
// FIXME: Do more extensive tests (ex: create multiple, delete, recreate;
// create multiple, check the amount of images and paths, etc..)
func TestGraphCreate(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
img, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
if err != nil {
t.Fatal(err)
}
if err := image.ValidateID(img.ID); err != nil {
t.Fatal(err)
}
if img.Comment != "Testing" {
t.Fatalf("Wrong comment: should be '%s', not '%s'", "Testing", img.Comment)
}
if img.DockerVersion != dockerversion.Version {
t.Fatalf("Wrong docker_version: should be '%s', not '%s'", dockerversion.Version, img.DockerVersion)
}
images := graph.Map()
if l := len(images); l != 1 {
t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l)
}
if images[img.ID] == nil {
t.Fatalf("Could not find image with id %s", img.ID)
}
}
func TestRegister(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
image := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "testing",
Created: time.Now(),
}
err = graph.Register(v1Descriptor{image}, archive)
if err != nil {
t.Fatal(err)
}
images := graph.Map()
if l := len(images); l != 1 {
t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l)
}
if resultImg, err := graph.Get(image.ID); err != nil {
t.Fatal(err)
} else {
if resultImg.ID != image.ID {
t.Fatalf("Wrong image ID. Should be '%s', not '%s'", image.ID, resultImg.ID)
}
if resultImg.Comment != image.Comment {
t.Fatalf("Wrong image comment. Should be '%s', not '%s'", image.Comment, resultImg.Comment)
}
}
}
// Test that an image can be deleted by its shorthand prefix
func TestDeletePrefix(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
img := createTestImage(graph, t)
if err := graph.Delete(stringid.TruncateID(img.ID)); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 0)
}
func TestDelete(t *testing.T) {
graph, _ := tempGraph(t)
defer nukeGraph(graph)
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 0)
img, err := graph.Create(archive, "", "", "Bla bla", "", nil, nil)
if err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 1)
if err := graph.Delete(img.ID); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 0)
archive, err = fakeTar()
if err != nil {
t.Fatal(err)
}
// Test 2 create (same name) / 1 delete
img1, err := graph.Create(archive, "", "", "Testing", "", nil, nil)
if err != nil {
t.Fatal(err)
}
archive, err = fakeTar()
if err != nil {
t.Fatal(err)
}
if _, err = graph.Create(archive, "", "", "Testing", "", nil, nil); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 2)
if err := graph.Delete(img1.ID); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 1)
// Test delete wrong name
if err := graph.Delete("Not_foo"); err == nil {
t.Fatalf("Deleting wrong ID should return an error")
}
assertNImages(graph, t, 1)
archive, err = fakeTar()
if err != nil {
t.Fatal(err)
}
// Test delete twice (pull -> rm -> pull -> rm)
if err := graph.Register(v1Descriptor{img1}, archive); err != nil {
t.Fatal(err)
}
if err := graph.Delete(img1.ID); err != nil {
t.Fatal(err)
}
assertNImages(graph, t, 1)
}
func TestByParent(t *testing.T) {
archive1, _ := fakeTar()
archive2, _ := fakeTar()
archive3, _ := fakeTar()
graph, _ := tempGraph(t)
defer nukeGraph(graph)
parentImage := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "parent",
Created: time.Now(),
Parent: "",
}
childImage1 := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "child1",
Created: time.Now(),
Parent: parentImage.ID,
}
childImage2 := &image.Image{
ID: stringid.GenerateNonCryptoID(),
Comment: "child2",
Created: time.Now(),
Parent: parentImage.ID,
}
err := graph.Register(v1Descriptor{parentImage}, archive1)
if err != nil {
t.Fatal(err)
}
err = graph.Register(v1Descriptor{childImage1}, archive2)
if err != nil {
t.Fatal(err)
}
err = graph.Register(v1Descriptor{childImage2}, archive3)
if err != nil {
t.Fatal(err)
}
byParent := graph.ByParent()
numChildren := len(byParent[parentImage.ID])
if numChildren != 2 {
t.Fatalf("Expected 2 children, found %d", numChildren)
}
}
func createTestImage(graph *Graph, t *testing.T) *image.Image {
archive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
img, err := graph.Create(archive, "", "", "Test image", "", nil, nil)
if err != nil {
t.Fatal(err)
}
return img
}
func assertNImages(graph *Graph, t *testing.T, n int) {
images := graph.Map()
if actualN := len(images); actualN != n {
t.Fatalf("Expected %d images, found %d", n, actualN)
}
}
func tempGraph(t *testing.T) (*Graph, graphdriver.Driver) {
tmp, err := ioutil.TempDir("", "docker-graph-")
if err != nil {
t.Fatal(err)
}
driver, err := graphdriver.New(tmp, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
graph, err := NewGraph(tmp, driver, nil, nil)
if err != nil {
t.Fatal(err)
}
return graph, driver
}
func nukeGraph(graph *Graph) {
graph.driver.Cleanup()
os.RemoveAll(graph.root)
}

8
graph/graph_unix.go
View File

@ -1,8 +0,0 @@
// +build !windows
package graph
// allowBaseParentImage allows images to define a custom parent that is not
// transported with push/pull but already included with the installation.
// Only used in Windows.
const allowBaseParentImage = false

8
graph/graph_windows.go
View File

@ -1,8 +0,0 @@
// +build windows
package graph
// allowBaseParentImage allows images to define a custom parent that is not
// transported with push/pull but already included with the installation.
// Only used in Windows.
const allowBaseParentImage = true

119
graph/history.go
View File

@ -1,119 +0,0 @@
package graph
import (
"fmt"
"strings"
"github.com/docker/docker/api/types"
"github.com/docker/docker/image"
"github.com/docker/docker/utils"
)
// walkHistory calls the handler function for each image in the
// provided images lineage starting from immediate parent.
func (graph *Graph) walkHistory(img *image.Image, handler func(image.Image) error) (err error) {
currentImg := img
for currentImg != nil {
if handler != nil {
if err := handler(*currentImg); err != nil {
return err
}
}
currentImg, err = graph.GetParent(currentImg)
if err != nil {
return fmt.Errorf("Error while getting parent image: %v", err)
}
}
return nil
}
// depth returns the number of parents for the current image
func (graph *Graph) depth(img *image.Image) (int, error) {
var (
count = 0
parent = img
err error
)
for parent != nil {
count++
if parent, err = graph.GetParent(parent); err != nil {
return -1, err
}
}
return count, nil
}
// Set the max depth to the aufs default that most kernels are compiled with.
// For more information see: http://sourceforge.net/p/aufs/aufs3-standalone/ci/aufs3.12/tree/config.mk
const maxImageDepth = 127
// CheckDepth returns an error if the depth of an image, as returned
// by ImageDepth, is too large to support creating a container from it
// on this daemon.
func (graph *Graph) CheckDepth(img *image.Image) error {
// We add 2 layers to the depth because the container's rw and
// init layer add to the restriction
depth, err := graph.depth(img)
if err != nil {
return err
}
if depth+2 >= maxImageDepth {
return fmt.Errorf("Cannot create container with more than %d parents", maxImageDepth)
}
return nil
}
// History returns a slice of ImageHistory structures for the specified image
// name by walking the image lineage.
func (s *TagStore) History(name string) ([]*types.ImageHistory, error) {
foundImage, err := s.LookupImage(name)
if err != nil {
return nil, err
}
lookupMap := make(map[string][]string)
for name, repository := range s.Repositories {
for tag, id := range repository {
// If the ID already has a reverse lookup, do not update it unless for "latest"
if _, exists := lookupMap[id]; !exists {
lookupMap[id] = []string{}
}
lookupMap[id] = append(lookupMap[id], utils.ImageReference(name, tag))
}
}
history := []*types.ImageHistory{}
err = s.graph.walkHistory(foundImage, func(img image.Image) error {
history = append(history, &types.ImageHistory{
ID: img.ID,
Created: img.Created.Unix(),
CreatedBy: strings.Join(img.ContainerConfig.Cmd.Slice(), " "),
Tags: lookupMap[img.ID],
Size: img.Size,
Comment: img.Comment,
})
return nil
})
return history, err
}
// GetParent returns the parent image for the specified image.
func (graph *Graph) GetParent(img *image.Image) (*image.Image, error) {
if img.Parent == "" {
return nil, nil
}
return graph.Get(img.Parent)
}
// getParentsSize returns the combined size of all parent images. If there is
// no parent image or it's unavailable, it returns 0.
func (graph *Graph) getParentsSize(img *image.Image) int64 {
parentImage, err := graph.GetParent(img)
if err != nil || parentImage == nil {
return 0
}
return parentImage.Size + graph.getParentsSize(parentImage)
}

73
graph/import.go
View File

@ -1,73 +0,0 @@
package graph
import (
"io"
"net/http"
"net/url"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/runconfig"
)
// Import imports an image, getting the archived layer data either from
// inConfig (if src is "-"), or from a URI specified in src. Progress output is
// written to outStream. Repository and tag names can optionally be given in
// the repo and tag arguments, respectively.
func (s *TagStore) Import(src string, repo string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, containerConfig *runconfig.Config) error {
var (
sf = streamformatter.NewJSONStreamFormatter()
archive io.ReadCloser
resp *http.Response
)
if src == "-" {
archive = inConfig
} else {
inConfig.Close()
u, err := url.Parse(src)
if err != nil {
return err
}
if u.Scheme == "" {
u.Scheme = "http"
u.Host = src
u.Path = ""
}
outStream.Write(sf.FormatStatus("", "Downloading from %s", u))
resp, err = httputils.Download(u.String())
if err != nil {
return err
}
progressReader := progressreader.New(progressreader.Config{
In: resp.Body,
Out: outStream,
Formatter: sf,
Size: resp.ContentLength,
NewLines: true,
ID: "",
Action: "Importing",
})
archive = progressReader
}
defer archive.Close()
if len(msg) == 0 {
msg = "Imported from " + src
}
img, err := s.graph.Create(archive, "", "", msg, "", nil, containerConfig)
if err != nil {
return err
}
// Optionally register the image at REPO/TAG
if repo != "" {
if err := s.Tag(repo, tag, img.ID, true); err != nil {
return err
}
}
outStream.Write(sf.FormatStatus("", img.ID))
s.eventsService.Log("import", img.ID, "")
return nil
}

185
graph/list.go
View File

@ -1,185 +0,0 @@
package graph
import (
"fmt"
"path"
"sort"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/types"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/parsers/filters"
"github.com/docker/docker/utils"
)
var acceptedImageFilterTags = map[string]struct{}{
"dangling": {},
"label": {},
}
// byCreated is a temporary type used to sort a list of images by creation
// time.
type byCreated []*types.Image
func (r byCreated) Len() int { return len(r) }
func (r byCreated) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created }
// Images returns a filtered list of images. filterArgs is a JSON-encoded set
// of filter arguments which will be interpreted by pkg/parsers/filters.
// filter is a shell glob string applied to repository names. The argument
// named all controls whether all images in the graph are filtered, or just
// the heads.
func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, error) {
var (
allImages map[string]*image.Image
err error
filtTagged = true
filtLabel = false
)
imageFilters, err := filters.FromParam(filterArgs)
if err != nil {
return nil, err
}
for name := range imageFilters {
if _, ok := acceptedImageFilterTags[name]; !ok {
return nil, fmt.Errorf("Invalid filter '%s'", name)
}
}
if i, ok := imageFilters["dangling"]; ok {
for _, value := range i {
if v := strings.ToLower(value); v == "true" {
filtTagged = false
} else if v != "false" {
return nil, fmt.Errorf("Invalid filter 'dangling=%s'", v)
}
}
}
_, filtLabel = imageFilters["label"]
if all && filtTagged {
allImages = s.graph.Map()
} else {
allImages = s.graph.heads()
}
lookup := make(map[string]*types.Image)
s.Lock()
for repoName, repository := range s.Repositories {
filterTagName := ""
if filter != "" {
filterName := filter
// Test if the tag was in there, if yes, get the name
if strings.Contains(filterName, ":") {
filterWithTag := strings.Split(filter, ":")
filterName = filterWithTag[0]
filterTagName = filterWithTag[1]
}
if match, _ := path.Match(filterName, repoName); !match {
continue
}
if filterTagName != "" {
if _, ok := repository[filterTagName]; !ok {
continue
}
}
}
for ref, id := range repository {
imgRef := utils.ImageReference(repoName, ref)
if !strings.Contains(imgRef, filterTagName) {
continue
}
image, err := s.graph.Get(id)
if err != nil {
logrus.Warnf("couldn't load %s from %s: %s", id, imgRef, err)
continue
}
if lImage, exists := lookup[id]; exists {
if filtTagged {
if utils.DigestReference(ref) {
lImage.RepoDigests = append(lImage.RepoDigests, imgRef)
} else { // Tag Ref.
lImage.RepoTags = append(lImage.RepoTags, imgRef)
}
}
} else {
// get the boolean list for if only the untagged images are requested
delete(allImages, id)
if len(imageFilters["label"]) > 0 {
if image.Config == nil {
// Very old image that do not have image.Config (or even labels)
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", image.Config.Labels) {
continue
}
}
if filtTagged {
newImage := newImage(image, s.graph.getParentsSize(image))
if utils.DigestReference(ref) {
newImage.RepoTags = []string{}
newImage.RepoDigests = []string{imgRef}
} else {
newImage.RepoTags = []string{imgRef}
newImage.RepoDigests = []string{}
}
lookup[id] = newImage
}
}
}
}
s.Unlock()
images := []*types.Image{}
for _, value := range lookup {
images = append(images, value)
}
// Display images which aren't part of a repository/tag
if filter == "" || filtLabel {
for _, image := range allImages {
if len(imageFilters["label"]) > 0 {
if image.Config == nil {
// Very old image that do not have image.Config (or even labels)
continue
}
// We are now sure image.Config is not nil
if !imageFilters.MatchKVList("label", image.Config.Labels) {
continue
}
}
newImage := newImage(image, s.graph.getParentsSize(image))
newImage.RepoTags = []string{"<none>:<none>"}
newImage.RepoDigests = []string{"<none>@<none>"}
images = append(images, newImage)
}
}
sort.Sort(sort.Reverse(byCreated(images)))
return images, nil
}
func newImage(image *image.Image, parentSize int64) *types.Image {
newImage := new(types.Image)
newImage.ParentID = image.Parent
newImage.ID = image.ID
newImage.Created = image.Created.Unix()
newImage.Size = image.Size
newImage.VirtualSize = parentSize + image.Size
if image.Config != nil {
newImage.Labels = image.Config.Labels
}
return newImage
}

134
graph/load.go
View File

@ -1,134 +0,0 @@
// +build linux windows
package graph
import (
"encoding/json"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/chrootarchive"
)
// Load uploads a set of images into the repository. This is the complementary of ImageExport.
// The input stream is an uncompressed tar ball containing images and metadata.
func (s *TagStore) Load(inTar io.ReadCloser, outStream io.Writer) error {
tmpImageDir, err := ioutil.TempDir("", "docker-import-")
if err != nil {
return err
}
defer os.RemoveAll(tmpImageDir)
var (
repoDir = filepath.Join(tmpImageDir, "repo")
)
if err := os.Mkdir(repoDir, os.ModeDir); err != nil {
return err
}
images := s.graph.Map()
excludes := make([]string, len(images))
i := 0
for k := range images {
excludes[i] = k
i++
}
if err := chrootarchive.Untar(inTar, repoDir, &archive.TarOptions{ExcludePatterns: excludes}); err != nil {
return err
}
dirs, err := ioutil.ReadDir(repoDir)
if err != nil {
return err
}
for _, d := range dirs {
if d.IsDir() {
if err := s.recursiveLoad(d.Name(), tmpImageDir); err != nil {
return err
}
}
}
reposJSONFile, err := os.Open(filepath.Join(tmpImageDir, "repo", "repositories"))
if err != nil {
if !os.IsNotExist(err) {
return err
}
return nil
}
defer reposJSONFile.Close()
repositories := map[string]repository{}
if err := json.NewDecoder(reposJSONFile).Decode(&repositories); err != nil {
return err
}
for imageName, tagMap := range repositories {
for tag, address := range tagMap {
if err := s.setLoad(imageName, tag, address, true, outStream); err != nil {
return err
}
}
}
return nil
}
func (s *TagStore) recursiveLoad(address, tmpImageDir string) error {
if _, err := s.LookupImage(address); err != nil {
logrus.Debugf("Loading %s", address)
imageJSON, err := ioutil.ReadFile(filepath.Join(tmpImageDir, "repo", address, "json"))
if err != nil {
logrus.Debugf("Error reading json: %v", err)
return err
}
layer, err := os.Open(filepath.Join(tmpImageDir, "repo", address, "layer.tar"))
if err != nil {
logrus.Debugf("Error reading embedded tar: %v", err)
return err
}
img, err := image.NewImgJSON(imageJSON)
if err != nil {
logrus.Debugf("Error unmarshalling json: %v", err)
return err
}
if err := image.ValidateID(img.ID); err != nil {
logrus.Debugf("Error validating ID: %v", err)
return err
}
// ensure no two downloads of the same layer happen at the same time
poolKey := "layer:" + img.ID
broadcaster, found := s.poolAdd("pull", poolKey)
if found {
logrus.Debugf("Image (id: %s) load is already running, waiting", img.ID)
return broadcaster.Wait()
}
defer s.poolRemove("pull", poolKey)
if img.Parent != "" {
if !s.graph.Exists(img.Parent) {
if err := s.recursiveLoad(img.Parent, tmpImageDir); err != nil {
return err
}
}
}
if err := s.graph.Register(v1Descriptor{img}, layer); err != nil {
return err
}
logrus.Debugf("Completed processing %s", address)
return nil
}
logrus.Debugf("already loaded %s", address)
return nil
}

14
graph/load_unsupported.go
View File

@ -1,14 +0,0 @@
// +build !linux,!windows
package graph
import (
"fmt"
"io"
)
// Load method is implemented here for non-linux and non-windows platforms and
// may return an error indicating that image load is not supported on other platforms.
func (s *TagStore) Load(inTar io.ReadCloser, outStream io.Writer) error {
return fmt.Errorf("Load is not supported on this platform")
}

44
graph/pools_test.go
View File

@ -1,44 +0,0 @@
package graph
import (
"testing"
"github.com/docker/docker/pkg/broadcaster"
"github.com/docker/docker/pkg/reexec"
)
func init() {
reexec.Init()
}
func TestPools(t *testing.T) {
s := &TagStore{
pullingPool: make(map[string]*broadcaster.Buffered),
pushingPool: make(map[string]*broadcaster.Buffered),
}
if _, found := s.poolAdd("pull", "test1"); found {
t.Fatal("Expected pull test1 not to be in progress")
}
if _, found := s.poolAdd("pull", "test2"); found {
t.Fatal("Expected pull test2 not to be in progress")
}
if _, found := s.poolAdd("push", "test1"); !found {
t.Fatalf("Expected pull test1 to be in progress`")
}
if _, found := s.poolAdd("pull", "test1"); !found {
t.Fatalf("Expected pull test1 to be in progress`")
}
if err := s.poolRemove("pull", "test2"); err != nil {
t.Fatal(err)
}
if err := s.poolRemove("pull", "test2"); err != nil {
t.Fatal(err)
}
if err := s.poolRemove("pull", "test1"); err != nil {
t.Fatal(err)
}
if err := s.poolRemove("push", "test1"); err != nil {
t.Fatal(err)
}
}

157
graph/pull.go
View File

@ -1,157 +0,0 @@
package graph
import (
"fmt"
"io"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/cliconfig"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
)
// ImagePullConfig stores pull configuration.
type ImagePullConfig struct {
// MetaHeaders stores HTTP headers with metadata about the image
// (DockerHeaders with prefix X-Meta- in the request).
MetaHeaders map[string][]string
// AuthConfig holds authentication credentials for authenticating with
// the registry.
AuthConfig *cliconfig.AuthConfig
// OutStream is the output writer for showing the status of the pull
// operation.
OutStream io.Writer
}
// puller is an interface that abstracts pulling for different API versions.
type puller interface {
// Pull tries to pull the image referenced by `tag`
// Pull returns an error if any, as well as a boolean that determines whether to retry Pull on the next configured endpoint.
//
// TODO(tiborvass): have Pull() take a reference to repository + tag, so that the puller itself is repository-agnostic.
Pull(tag string) (fallback bool, err error)
}
// newPuller returns a Puller interface that will pull from either a v1 or v2
// registry. The endpoint argument contains a Version field that determines
// whether a v1 or v2 puller will be created. The other parameters are passed
// through to the underlying puller implementation for use during the actual
// pull operation.
func newPuller(s *TagStore, endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePullConfig *ImagePullConfig, sf *streamformatter.StreamFormatter) (puller, error) {
switch endpoint.Version {
case registry.APIVersion2:
return &v2Puller{
TagStore: s,
endpoint: endpoint,
config: imagePullConfig,
sf: sf,
repoInfo: repoInfo,
}, nil
case registry.APIVersion1:
return &v1Puller{
TagStore: s,
endpoint: endpoint,
config: imagePullConfig,
sf: sf,
repoInfo: repoInfo,
}, nil
}
return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
}
// Pull initiates a pull operation. image is the repository name to pull, and
// tag may be either empty, or indicate a specific tag to pull.
func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConfig) error {
var sf = streamformatter.NewJSONStreamFormatter()
// Resolve the Repository name from fqn to RepositoryInfo
repoInfo, err := s.registryService.ResolveRepository(image)
if err != nil {
return err
}
// makes sure name is not empty or `scratch`
if err := validateRepoName(repoInfo.LocalName); err != nil {
return err
}
endpoints, err := s.registryService.LookupPullEndpoints(repoInfo.CanonicalName)
if err != nil {
return err
}
logName := repoInfo.LocalName
if tag != "" {
logName = utils.ImageReference(logName, tag)
}
var (
// use a slice to append the error strings and return a joined string to caller
errors []string
// discardNoSupportErrors is used to track whether an endpoint encountered an error of type registry.ErrNoSupport
// By default it is false, which means that if a ErrNoSupport error is encountered, it will be saved in errors.
// As soon as another kind of error is encountered, discardNoSupportErrors is set to true, avoiding the saving of
// any subsequent ErrNoSupport errors in errors.
// It's needed for pull-by-digest on v1 endpoints: if there are only v1 endpoints configured, the error should be
// returned and displayed, but if there was a v2 endpoint which supports pull-by-digest, then the last relevant
// error is the ones from v2 endpoints not v1.
discardNoSupportErrors bool
)
for _, endpoint := range endpoints {
logrus.Debugf("Trying to pull %s from %s %s", repoInfo.LocalName, endpoint.URL, endpoint.Version)
puller, err := newPuller(s, endpoint, repoInfo, imagePullConfig, sf)
if err != nil {
errors = append(errors, err.Error())
continue
}
if fallback, err := puller.Pull(tag); err != nil {
if fallback {
if _, ok := err.(registry.ErrNoSupport); !ok {
// Because we found an error that's not ErrNoSupport, discard all subsequent ErrNoSupport errors.
discardNoSupportErrors = true
// append subsequent errors
errors = append(errors, err.Error())
} else if !discardNoSupportErrors {
// Save the ErrNoSupport error, because it's either the first error or all encountered errors
// were also ErrNoSupport errors.
// append subsequent errors
errors = append(errors, err.Error())
}
continue
}
errors = append(errors, err.Error())
logrus.Debugf("Not continuing with error: %v", fmt.Errorf(strings.Join(errors, "\n")))
if len(errors) > 0 {
return fmt.Errorf(strings.Join(errors, "\n"))
}
}
s.eventsService.Log("pull", logName, "")
return nil
}
if len(errors) == 0 {
return fmt.Errorf("no endpoints found for %s", image)
}
if len(errors) > 0 {
return fmt.Errorf(strings.Join(errors, "\n"))
}
return nil
}
// writeStatus writes a status message to out. If layersDownloaded is true, the
// status message indicates that a newer image was downloaded. Otherwise, it
// indicates that the image is up to date. requestedTag is the tag the message
// will refer to.
func writeStatus(requestedTag string, out io.Writer, sf *streamformatter.StreamFormatter, layersDownloaded bool) {
if layersDownloaded {
out.Write(sf.FormatStatus("", "Status: Downloaded newer image for %s", requestedTag))
} else {
out.Write(sf.FormatStatus("", "Status: Image is up to date for %s", requestedTag))
}
}

350
graph/pull_v1.go
View File

@ -1,350 +0,0 @@
package graph
import (
"errors"
"fmt"
"io"
"net"
"net/url"
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
)
type v1Puller struct {
*TagStore
endpoint registry.APIEndpoint
config *ImagePullConfig
sf *streamformatter.StreamFormatter
repoInfo *registry.RepositoryInfo
session *registry.Session
}
func (p *v1Puller) Pull(tag string) (fallback bool, err error) {
if utils.DigestReference(tag) {
// Allowing fallback, because HTTPS v1 is before HTTP v2
return true, registry.ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}
}
tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name)
if err != nil {
return false, err
}
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
tr := transport.NewTransport(
// TODO(tiborvass): was ReceiveTimeout
registry.NewTransport(tlsConfig),
registry.DockerHeaders(p.config.MetaHeaders)...,
)
client := registry.HTTPClient(tr)
v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
if err != nil {
logrus.Debugf("Could not get v1 endpoint: %v", err)
return true, err
}
p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
if err != nil {
// TODO(dmcgowan): Check if should fallback
logrus.Debugf("Fallback from error: %s", err)
return true, err
}
if err := p.pullRepository(tag); err != nil {
// TODO(dmcgowan): Check if should fallback
return false, err
}
out := p.config.OutStream
out.Write(p.sf.FormatStatus("", "%s: this image was pulled from a legacy registry. Important: This registry version will not be supported in future versions of docker.", p.repoInfo.CanonicalName))
return false, nil
}
func (p *v1Puller) pullRepository(askedTag string) error {
out := p.config.OutStream
out.Write(p.sf.FormatStatus("", "Pulling repository %s", p.repoInfo.CanonicalName))
repoData, err := p.session.GetRepositoryData(p.repoInfo.RemoteName)
if err != nil {
if strings.Contains(err.Error(), "HTTP code: 404") {
return fmt.Errorf("Error: image %s not found", utils.ImageReference(p.repoInfo.RemoteName, askedTag))
}
// Unexpected HTTP error
return err
}
logrus.Debugf("Retrieving the tag list")
tagsList := make(map[string]string)
if askedTag == "" {
tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo.RemoteName)
} else {
var tagID string
tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo.RemoteName, askedTag)
tagsList[askedTag] = tagID
}
if err != nil {
if err == registry.ErrRepoNotFound && askedTag != "" {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName)
}
logrus.Errorf("unable to get remote tags: %s", err)
return err
}
for tag, id := range tagsList {
repoData.ImgList[id] = &registry.ImgData{
ID: id,
Tag: tag,
Checksum: "",
}
}
logrus.Debugf("Registering tags")
// If no tag has been specified, pull them all
if askedTag == "" {
for tag, id := range tagsList {
repoData.ImgList[id].Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
id, exists := tagsList[askedTag]
if !exists {
return fmt.Errorf("Tag %s not found in repository %s", askedTag, p.repoInfo.CanonicalName)
}
repoData.ImgList[id].Tag = askedTag
}
errors := make(chan error)
layersDownloaded := false
imgIDs := []string{}
sessionID := p.session.ID()
defer func() {
p.graph.Release(sessionID, imgIDs...)
}()
for _, imgData := range repoData.ImgList {
downloadImage := func(img *registry.ImgData) {
if askedTag != "" && img.Tag != askedTag {
errors <- nil
return
}
if img.Tag == "" {
logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
errors <- nil
return
}
if err := image.ValidateID(img.ID); err != nil {
errors <- err
return
}
// ensure no two downloads of the same image happen at the same time
poolKey := "img:" + img.ID
broadcaster, found := p.poolAdd("pull", poolKey)
broadcaster.Add(out)
if found {
errors <- broadcaster.Wait()
return
}
defer p.poolRemove("pull", poolKey)
// we need to retain it until tagging
p.graph.Retain(sessionID, img.ID)
imgIDs = append(imgIDs, img.ID)
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, p.repoInfo.CanonicalName), nil))
success := false
var lastErr, err error
var isDownloaded bool
for _, ep := range p.repoInfo.Index.Mirrors {
ep += "v1/"
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil))
if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil {
// Don't report errors when pulling from mirrors.
logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err)
continue
}
layersDownloaded = layersDownloaded || isDownloaded
success = true
break
}
if !success {
for _, ep := range repoData.Endpoints {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.CanonicalName, ep), nil))
if isDownloaded, err = p.pullImage(broadcaster, img.ID, ep); err != nil {
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.CanonicalName, ep, err), nil))
continue
}
layersDownloaded = layersDownloaded || isDownloaded
success = true
break
}
}
if !success {
err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.CanonicalName, lastErr)
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil))
errors <- err
broadcaster.CloseWithError(err)
return
}
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
errors <- nil
}
go downloadImage(imgData)
}
var lastError error
for i := 0; i < len(repoData.ImgList); i++ {
if err := <-errors; err != nil {
lastError = err
}
}
if lastError != nil {
return lastError
}
for tag, id := range tagsList {
if askedTag != "" && tag != askedTag {
continue
}
if err := p.Tag(p.repoInfo.LocalName, tag, id, true); err != nil {
return err
}
}
requestedTag := p.repoInfo.LocalName
if len(askedTag) > 0 {
requestedTag = utils.ImageReference(p.repoInfo.LocalName, askedTag)
}
writeStatus(requestedTag, out, p.sf, layersDownloaded)
return nil
}
func (p *v1Puller) pullImage(out io.Writer, imgID, endpoint string) (layersDownloaded bool, err error) {
var history []string
history, err = p.session.GetRemoteHistory(imgID, endpoint)
if err != nil {
return false, err
}
out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pulling dependent layers", nil))
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
sessionID := p.session.ID()
// As imgID has been retained in pullRepository, no need to retain again
p.graph.Retain(sessionID, history[1:]...)
defer p.graph.Release(sessionID, history[1:]...)
layersDownloaded = false
for i := len(history) - 1; i >= 0; i-- {
id := history[i]
// ensure no two downloads of the same layer happen at the same time
poolKey := "layer:" + id
broadcaster, found := p.poolAdd("pull", poolKey)
broadcaster.Add(out)
if found {
logrus.Debugf("Image (id: %s) pull is already running, skipping", id)
err = broadcaster.Wait()
if err != nil {
return layersDownloaded, err
}
continue
}
// This must use a closure so it captures the value of err when
// the function returns, not when the 'defer' is evaluated.
defer func() {
p.poolRemoveWithError("pull", poolKey, err)
}()
if !p.graph.Exists(id) {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Pulling metadata", nil))
var (
imgJSON []byte
imgSize int64
err error
img *image.Image
)
retries := 5
for j := 1; j <= retries; j++ {
imgJSON, imgSize, err = p.session.GetRemoteImageJSON(id, endpoint)
if err != nil && j == retries {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
return layersDownloaded, err
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
}
img, err = image.NewImgJSON(imgJSON)
layersDownloaded = true
if err != nil && j == retries {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
return layersDownloaded, fmt.Errorf("Failed to parse json: %s", err)
} else if err != nil {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else {
break
}
}
for j := 1; j <= retries; j++ {
// Get the layer
status := "Pulling fs layer"
if j > 1 {
status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
}
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), status, nil))
layer, err := p.session.GetRemoteImageLayer(img.ID, endpoint, imgSize)
if uerr, ok := err.(*url.Error); ok {
err = uerr.Err
}
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
return layersDownloaded, err
}
layersDownloaded = true
defer layer.Close()
err = p.graph.Register(v1Descriptor{img},
progressreader.New(progressreader.Config{
In: layer,
Out: broadcaster,
Formatter: p.sf,
Size: imgSize,
NewLines: false,
ID: stringid.TruncateID(id),
Action: "Downloading",
}))
if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
time.Sleep(time.Duration(j) * 500 * time.Millisecond)
continue
} else if err != nil {
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Error downloading dependent layers", nil))
return layersDownloaded, err
} else {
break
}
}
}
broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Download complete", nil))
broadcaster.Close()
}
return layersDownloaded, nil
}

728
graph/pull_v2.go
View File

@ -1,728 +0,0 @@
package graph
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/broadcaster"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
"golang.org/x/net/context"
)
type v2Puller struct {
*TagStore
endpoint registry.APIEndpoint
config *ImagePullConfig
sf *streamformatter.StreamFormatter
repoInfo *registry.RepositoryInfo
repo distribution.Repository
sessionID string
}
func (p *v2Puller) Pull(tag string) (fallback bool, err error) {
// TODO(tiborvass): was ReceiveTimeout
p.repo, err = newV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
if err != nil {
logrus.Warnf("Error getting v2 registry: %v", err)
return true, err
}
p.sessionID = stringid.GenerateRandomID()
if err := p.pullV2Repository(tag); err != nil {
if registry.ContinueOnError(err) {
logrus.Debugf("Error trying v2 registry: %v", err)
return true, err
}
return false, err
}
return false, nil
}
func (p *v2Puller) pullV2Repository(tag string) (err error) {
var tags []string
taggedName := p.repoInfo.LocalName
if len(tag) > 0 {
tags = []string{tag}
taggedName = utils.ImageReference(p.repoInfo.LocalName, tag)
} else {
var err error
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return err
}
tags, err = manSvc.Tags()
if err != nil {
return err
}
}
poolKey := "v2:" + taggedName
broadcaster, found := p.poolAdd("pull", poolKey)
broadcaster.Add(p.config.OutStream)
if found {
// Another pull of the same repository is already taking place; just wait for it to finish
return broadcaster.Wait()
}
// This must use a closure so it captures the value of err when the
// function returns, not when the 'defer' is evaluated.
defer func() {
p.poolRemoveWithError("pull", poolKey, err)
}()
var layersDownloaded bool
for _, tag := range tags {
// pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
// TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
pulledNew, err := p.pullV2Tag(broadcaster, tag, taggedName)
if err != nil {
return err
}
layersDownloaded = layersDownloaded || pulledNew
}
writeStatus(taggedName, broadcaster, p.sf, layersDownloaded)
return nil
}
// downloadInfo is used to pass information from download to extractor
type downloadInfo struct {
img contentAddressableDescriptor
imgIndex int
tmpFile *os.File
digest digest.Digest
layer distribution.ReadSeekCloser
size int64
err chan error
poolKey string
broadcaster *broadcaster.Buffered
}
// contentAddressableDescriptor is used to pass image data from a manifest to the
// graph.
type contentAddressableDescriptor struct {
id string
parent string
strongID digest.Digest
compatibilityID string
config []byte
v1Compatibility []byte
}
func newContentAddressableImage(v1Compatibility []byte, blobSum digest.Digest, parent digest.Digest) (contentAddressableDescriptor, error) {
img := contentAddressableDescriptor{
v1Compatibility: v1Compatibility,
}
var err error
img.config, err = image.MakeImageConfig(v1Compatibility, blobSum, parent)
if err != nil {
return img, err
}
img.strongID, err = image.StrongID(img.config)
if err != nil {
return img, err
}
unmarshalledConfig, err := image.NewImgJSON(v1Compatibility)
if err != nil {
return img, err
}
img.compatibilityID = unmarshalledConfig.ID
img.id = img.strongID.Hex()
return img, nil
}
// ID returns the actual ID to be used for the downloaded image. This may be
// a computed ID.
func (img contentAddressableDescriptor) ID() string {
return img.id
}
// Parent returns the parent ID to be used for the image. This may be a
// computed ID.
func (img contentAddressableDescriptor) Parent() string {
return img.parent
}
// MarshalConfig renders the image structure into JSON.
func (img contentAddressableDescriptor) MarshalConfig() ([]byte, error) {
return img.config, nil
}
type errVerification struct{}
func (errVerification) Error() string { return "verification failed" }
func (p *v2Puller) download(di *downloadInfo) {
logrus.Debugf("pulling blob %q to %s", di.digest, di.img.id)
blobs := p.repo.Blobs(context.Background())
desc, err := blobs.Stat(context.Background(), di.digest)
if err != nil {
logrus.Debugf("Error statting layer: %v", err)
di.err <- err
return
}
di.size = desc.Size
layerDownload, err := blobs.Open(context.Background(), di.digest)
if err != nil {
logrus.Debugf("Error fetching layer: %v", err)
di.err <- err
return
}
defer layerDownload.Close()
verifier, err := digest.NewDigestVerifier(di.digest)
if err != nil {
di.err <- err
return
}
reader := progressreader.New(progressreader.Config{
In: ioutil.NopCloser(io.TeeReader(layerDownload, verifier)),
Out: di.broadcaster,
Formatter: p.sf,
Size: di.size,
NewLines: false,
ID: stringid.TruncateID(di.img.id),
Action: "Downloading",
})
io.Copy(di.tmpFile, reader)
di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Verifying Checksum", nil))
if !verifier.Verified() {
err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest)
logrus.Error(err)
di.err <- err
return
}
di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Download complete", nil))
logrus.Debugf("Downloaded %s to tempfile %s", di.img.id, di.tmpFile.Name())
di.layer = layerDownload
di.err <- nil
}
func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (tagUpdated bool, err error) {
logrus.Debugf("Pulling tag from V2 registry: %q", tag)
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return false, err
}
unverifiedManifest, err := manSvc.GetByTag(tag)
if err != nil {
return false, err
}
if unverifiedManifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag %q", tag)
}
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifyManifest(unverifiedManifest, tag)
if err != nil {
return false, err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return false, err
}
imgs, err := p.getImageInfos(verifiedManifest)
if err != nil {
return false, err
}
out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
var downloads []*downloadInfo
var layerIDs []string
defer func() {
p.graph.Release(p.sessionID, layerIDs...)
for _, d := range downloads {
p.poolRemoveWithError("pull", d.poolKey, err)
if d.tmpFile != nil {
d.tmpFile.Close()
if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
}
}
}
}()
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
img := imgs[i]
p.graph.Retain(p.sessionID, img.id)
layerIDs = append(layerIDs, img.id)
p.graph.imageMutex.Lock(img.id)
// Check if exists
if p.graph.Exists(img.id) {
if err := p.validateImageInGraph(img.id, imgs, i); err != nil {
p.graph.imageMutex.Unlock(img.id)
return false, fmt.Errorf("image validation failed: %v", err)
}
logrus.Debugf("Image already exists: %s", img.id)
p.graph.imageMutex.Unlock(img.id)
continue
}
p.graph.imageMutex.Unlock(img.id)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil))
d := &downloadInfo{
img: img,
imgIndex: i,
poolKey: "v2layer:" + img.id,
digest: verifiedManifest.FSLayers[i].BlobSum,
// TODO: seems like this chan buffer solved hanging problem in go1.5,
// this can indicate some deeper problem that somehow we never take
// error from channel in loop below
err: make(chan error, 1),
}
tmpFile, err := ioutil.TempFile("", "GetImageBlob")
if err != nil {
return false, err
}
d.tmpFile = tmpFile
downloads = append(downloads, d)
broadcaster, found := p.poolAdd("pull", d.poolKey)
broadcaster.Add(out)
d.broadcaster = broadcaster
if found {
d.err <- nil
} else {
go p.download(d)
}
}
for _, d := range downloads {
if err := <-d.err; err != nil {
return false, err
}
if d.layer == nil {
// Wait for a different pull to download and extract
// this layer.
err = d.broadcaster.Wait()
if err != nil {
return false, err
}
continue
}
d.tmpFile.Seek(0, 0)
err := func() error {
reader := progressreader.New(progressreader.Config{
In: d.tmpFile,
Out: d.broadcaster,
Formatter: p.sf,
Size: d.size,
NewLines: false,
ID: stringid.TruncateID(d.img.id),
Action: "Extracting",
})
p.graph.imagesMutex.Lock()
defer p.graph.imagesMutex.Unlock()
p.graph.imageMutex.Lock(d.img.id)
defer p.graph.imageMutex.Unlock(d.img.id)
// Must recheck the data on disk if any exists.
// This protects against races where something
// else is written to the graph under this ID
// after attemptIDReuse.
if p.graph.Exists(d.img.id) {
if err := p.validateImageInGraph(d.img.id, imgs, d.imgIndex); err != nil {
return fmt.Errorf("image validation failed: %v", err)
}
}
if err := p.graph.register(d.img, reader); err != nil {
return err
}
if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil {
return err
}
if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil {
return err
}
return nil
}()
if err != nil {
return false, err
}
d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil))
d.broadcaster.Close()
tagUpdated = true
}
manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName)
if err != nil {
return false, err
}
// Check for new tag if no layers downloaded
if !tagUpdated {
repo, err := p.get(p.repoInfo.LocalName)
if err != nil {
return false, err
}
if repo != nil {
if _, exists := repo[tag]; !exists {
tagUpdated = true
}
} else {
tagUpdated = true
}
}
firstID := layerIDs[len(layerIDs)-1]
if utils.DigestReference(tag) {
// TODO(stevvooe): Ideally, we should always set the digest so we can
// use the digest whether we pull by it or not. Unfortunately, the tag
// store treats the digest as a separate tag, meaning there may be an
// untagged digest image that would seem to be dangling by a user.
if err = p.setDigest(p.repoInfo.LocalName, tag, firstID); err != nil {
return false, err
}
} else {
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil {
return false, err
}
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
}
return tagUpdated, nil
}
func verifyManifest(signedManifest *schema1.SignedManifest, tag string) (m *schema1.Manifest, err error) {
// If pull by digest, then verify the manifest digest. NOTE: It is
// important to do this first, before any other content validation. If the
// digest cannot be verified, don't even bother with those other things.
if manifestDigest, err := digest.ParseDigest(tag); err == nil {
verifier, err := digest.NewDigestVerifier(manifestDigest)
if err != nil {
return nil, err
}
payload, err := signedManifest.Payload()
if err != nil {
// If this failed, the signatures section was corrupted
// or missing. Treat the entire manifest as the payload.
payload = signedManifest.Raw
}
if _, err := verifier.Write(payload); err != nil {
return nil, err
}
if !verifier.Verified() {
err := fmt.Errorf("image verification failed for digest %s", manifestDigest)
logrus.Error(err)
return nil, err
}
var verifiedManifest schema1.Manifest
if err = json.Unmarshal(payload, &verifiedManifest); err != nil {
return nil, err
}
m = &verifiedManifest
} else {
m = &signedManifest.Manifest
}
if m.SchemaVersion != 1 {
return nil, fmt.Errorf("unsupported schema version %d for tag %q", m.SchemaVersion, tag)
}
if len(m.FSLayers) != len(m.History) {
return nil, fmt.Errorf("length of history not equal to number of layers for tag %q", tag)
}
if len(m.FSLayers) == 0 {
return nil, fmt.Errorf("no FSLayers in manifest for tag %q", tag)
}
return m, nil
}
// fixManifestLayers removes repeated layers from the manifest and checks the
// correctness of the parent chain.
func fixManifestLayers(m *schema1.Manifest) error {
images := make([]*image.Image, len(m.FSLayers))
for i := range m.FSLayers {
img, err := image.NewImgJSON([]byte(m.History[i].V1Compatibility))
if err != nil {
return err
}
images[i] = img
if err := image.ValidateID(img.ID); err != nil {
return err
}
}
if images[len(images)-1].Parent != "" && !allowBaseParentImage {
// Windows base layer can point to a base layer parent that is not in manifest.
return errors.New("Invalid parent ID in the base layer of the image.")
}
// check general duplicates to error instead of a deadlock
idmap := make(map[string]struct{})
var lastID string
for _, img := range images {
// skip IDs that appear after each other, we handle those later
if _, exists := idmap[img.ID]; img.ID != lastID && exists {
return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
}
lastID = img.ID
idmap[lastID] = struct{}{}
}
// backwards loop so that we keep the remaining indexes after removing items
for i := len(images) - 2; i >= 0; i-- {
if images[i].ID == images[i+1].ID { // repeated ID. remove and continue
m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
m.History = append(m.History[:i], m.History[i+1:]...)
} else if images[i].Parent != images[i+1].ID {
return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", images[i+1].ID, images[i].Parent)
}
}
return nil
}
// getImageInfos returns an imageinfo struct for every image in the manifest.
// These objects contain both calculated strongIDs and compatibilityIDs found
// in v1Compatibility object.
func (p *v2Puller) getImageInfos(m *schema1.Manifest) ([]contentAddressableDescriptor, error) {
imgs := make([]contentAddressableDescriptor, len(m.FSLayers))
var parent digest.Digest
for i := len(imgs) - 1; i >= 0; i-- {
var err error
imgs[i], err = newContentAddressableImage([]byte(m.History[i].V1Compatibility), m.FSLayers[i].BlobSum, parent)
if err != nil {
return nil, err
}
parent = imgs[i].strongID
}
p.attemptIDReuse(imgs)
// reset the base layer parent for windows
if allowBaseParentImage {
var base struct{ Parent string }
if err := json.Unmarshal(imgs[len(imgs)-1].v1Compatibility, &base); err != nil {
return nil, err
}
if base.Parent != "" {
imgs[len(imgs)-1].parent = base.Parent
}
}
return imgs, nil
}
// attemptIDReuse does a best attempt to match verified compatibilityIDs
// already in the graph with the computed strongIDs so we can keep using them.
// This process will never fail but may just return the strongIDs if none of
// the compatibilityIDs exists or can be verified. If the strongIDs themselves
// fail verification, we deterministically generate alternate IDs to use until
// we find one that's available or already exists with the correct data.
func (p *v2Puller) attemptIDReuse(imgs []contentAddressableDescriptor) {
// This function needs to be protected with a global lock, because it
// locks multiple IDs at once, and there's no good way to make sure
// the locking happens a deterministic order.
p.graph.imagesMutex.Lock()
defer p.graph.imagesMutex.Unlock()
idMap := make(map[string]struct{})
for _, img := range imgs {
idMap[img.id] = struct{}{}
idMap[img.compatibilityID] = struct{}{}
if p.graph.Exists(img.compatibilityID) {
if _, err := p.graph.generateV1CompatibilityChain(img.compatibilityID); err != nil {
logrus.Debugf("Migration v1Compatibility generation error: %v", err)
return
}
}
}
for id := range idMap {
p.graph.imageMutex.Lock(id)
defer p.graph.imageMutex.Unlock(id)
}
// continueReuse controls whether the function will try to find
// existing layers on disk under the old v1 IDs, to avoid repulling
// them. The hashes are checked to ensure these layers are okay to
// use. continueReuse starts out as true, but is set to false if
// the code encounters something that doesn't match the expected hash.
continueReuse := true
for i := len(imgs) - 1; i >= 0; i-- {
// TODO - (swernli:11-16-2015) Skipping content addressable IDs on
// Windows as a hack for TP4 compat. The correct fix is to ensure that
// Windows layers do not have anything in them that takes a dependency
// on the ID of the layer in the management client. This will be fixed
// in Windows post-TP4.
if runtime.GOOS == "windows" {
imgs[i].id = imgs[i].compatibilityID
}
if p.graph.Exists(imgs[i].id) {
// Found an image in the graph under the strongID. Validate the
// image before using it.
if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
continueReuse = false
logrus.Debugf("not using existing strongID: %v", err)
// The strong ID existed in the graph but didn't
// validate successfully. We can't use the strong ID
// because it didn't validate successfully. Treat the
// graph like a hash table with probing... compute
// SHA256(id) until we find an ID that either doesn't
// already exist in the graph, or has existing content
// that validates successfully.
for {
if err := p.tryNextID(imgs, i, idMap); err != nil {
logrus.Debug(err.Error())
} else {
break
}
}
}
continue
}
if continueReuse {
compatibilityID := imgs[i].compatibilityID
if err := p.validateImageInGraph(compatibilityID, imgs, i); err != nil {
logrus.Debugf("stopping ID reuse: %v", err)
continueReuse = false
} else {
// The compatibility ID exists in the graph and was
// validated. Use it.
imgs[i].id = compatibilityID
}
}
}
// fix up the parents of the images
for i := 0; i < len(imgs); i++ {
if i == len(imgs)-1 { // Base layer
imgs[i].parent = ""
} else {
imgs[i].parent = imgs[i+1].id
}
}
}
// validateImageInGraph checks that an image in the graph has the expected
// strongID. id is the entry in the graph to check, imgs is the slice of
// images being processed (for access to the parent), and i is the index
// into this slice which the graph entry should be checked against.
func (p *v2Puller) validateImageInGraph(id string, imgs []contentAddressableDescriptor, i int) error {
img, err := p.graph.Get(id)
if err != nil {
return fmt.Errorf("missing: %v", err)
}
if runtime.GOOS == "windows" {
// TODO - (swernli:11-16-2015) Skipping content addressable IDs on
// Windows as a hack for TP4 compat. The correct fix is to ensure that
// Windows layers do not have anything in them that takes a dependency
// on the ID of the layer in the management client. This will be fixed
// in Windows post-TP4.
return nil
}
layerID, err := p.graph.getLayerDigest(id)
if err != nil {
return fmt.Errorf("digest: %v", err)
}
var parentID digest.Digest
if i != len(imgs)-1 {
if img.Parent != imgs[i+1].id { // comparing that graph points to validated ID
return fmt.Errorf("parent: %v %v", img.Parent, imgs[i+1].id)
}
parentID = imgs[i+1].strongID
} else if img.Parent != "" {
return fmt.Errorf("unexpected parent: %v", img.Parent)
}
v1Config, err := p.graph.getV1CompatibilityConfig(img.ID)
if err != nil {
return fmt.Errorf("v1Compatibility: %v %v", img.ID, err)
}
json, err := image.MakeImageConfig(v1Config, layerID, parentID)
if err != nil {
return fmt.Errorf("make config: %v", err)
}
if dgst, err := image.StrongID(json); err == nil && dgst == imgs[i].strongID {
logrus.Debugf("Validated %v as %v", dgst, id)
} else {
return fmt.Errorf("digest mismatch: %v %v, error: %v", dgst, imgs[i].strongID, err)
}
// All clear
return nil
}
func (p *v2Puller) tryNextID(imgs []contentAddressableDescriptor, i int, idMap map[string]struct{}) error {
nextID, _ := digest.FromBytes([]byte(imgs[i].id))
imgs[i].id = nextID.Hex()
if _, exists := idMap[imgs[i].id]; !exists {
p.graph.imageMutex.Lock(imgs[i].id)
defer p.graph.imageMutex.Unlock(imgs[i].id)
}
if p.graph.Exists(imgs[i].id) {
if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
return fmt.Errorf("not using existing strongID permutation %s: %v", imgs[i].id, err)
}
}
return nil
}

195
graph/pull_v2_test.go
View File

@ -1,195 +0,0 @@
package graph
import (
"encoding/json"
"io/ioutil"
"reflect"
"strings"
"testing"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
)
// TestFixManifestLayers checks that fixManifestLayers removes a duplicate
// layer, and that it makes no changes to the manifest when called a second
// time, after the duplicate is removed.
func TestFixManifestLayers(t *testing.T) {
duplicateLayerManifest := schema1.Manifest{
FSLayers: []schema1.FSLayer{
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
},
History: []schema1.History{
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
},
}
duplicateLayerManifestExpectedOutput := schema1.Manifest{
FSLayers: []schema1.FSLayer{
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
},
History: []schema1.History{
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
},
}
if err := fixManifestLayers(&duplicateLayerManifest); err != nil {
t.Fatalf("unexpected error from fixManifestLayers: %v", err)
}
if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) {
t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest")
}
// Run fixManifestLayers again and confirm that it doesn't change the
// manifest (which no longer has duplicate layers).
if err := fixManifestLayers(&duplicateLayerManifest); err != nil {
t.Fatalf("unexpected error from fixManifestLayers: %v", err)
}
if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) {
t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest (second pass)")
}
}
// TestFixManifestLayersBaseLayerParent makes sure that fixManifestLayers fails
// if the base layer configuration specifies a parent.
func TestFixManifestLayersBaseLayerParent(t *testing.T) {
duplicateLayerManifest := schema1.Manifest{
FSLayers: []schema1.FSLayer{
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
},
History: []schema1.History{
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"parent\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
},
}
if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID in the base layer of the image.") {
t.Fatalf("expected an invalid parent ID error from fixManifestLayers")
}
}
// TestFixManifestLayersBadParent makes sure that fixManifestLayers fails
// if an image configuration specifies a parent that doesn't directly follow
// that (deduplicated) image in the image history.
func TestFixManifestLayersBadParent(t *testing.T) {
duplicateLayerManifest := schema1.Manifest{
FSLayers: []schema1.FSLayer{
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
},
History: []schema1.History{
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
},
}
if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID.") {
t.Fatalf("expected an invalid parent ID error from fixManifestLayers")
}
}
// TestValidateManifest verifies the validateManifest function
func TestValidateManifest(t *testing.T) {
expectedDigest := "sha256:02fee8c3220ba806531f606525eceb83f4feb654f62b207191b1c9209188dedd"
expectedFSLayer0 := digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
// Good manifest
goodManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/good_manifest")
if err != nil {
t.Fatal("error reading fixture:", err)
}
var goodSignedManifest schema1.SignedManifest
err = json.Unmarshal(goodManifestBytes, &goodSignedManifest)
if err != nil {
t.Fatal("error unmarshaling manifest:", err)
}
verifiedManifest, err := verifyManifest(&goodSignedManifest, expectedDigest)
if err != nil {
t.Fatal("validateManifest failed:", err)
}
if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
t.Fatal("unexpected FSLayer in good manifest")
}
// "Extra data" manifest
extraDataManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/extra_data_manifest")
if err != nil {
t.Fatal("error reading fixture:", err)
}
var extraDataSignedManifest schema1.SignedManifest
err = json.Unmarshal(extraDataManifestBytes, &extraDataSignedManifest)
if err != nil {
t.Fatal("error unmarshaling manifest:", err)
}
verifiedManifest, err = verifyManifest(&extraDataSignedManifest, expectedDigest)
if err != nil {
t.Fatal("validateManifest failed:", err)
}
if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
t.Fatal("unexpected FSLayer in extra data manifest")
}
// Bad manifest
badManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/bad_manifest")
if err != nil {
t.Fatal("error reading fixture:", err)
}
var badSignedManifest schema1.SignedManifest
err = json.Unmarshal(badManifestBytes, &badSignedManifest)
if err != nil {
t.Fatal("error unmarshaling manifest:", err)
}
verifiedManifest, err = verifyManifest(&badSignedManifest, expectedDigest)
if err == nil || !strings.HasPrefix(err.Error(), "image verification failed for digest") {
t.Fatal("expected validateManifest to fail with digest error")
}
// Manifest with no signature
expectedWholeFileDigest := "7ec3615a120efcdfc270e9c7ea4183330775a3e52a09e2efb194b9a7c18e5ff7"
noSignatureManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/no_signature_manifest")
if err != nil {
t.Fatal("error reading fixture:", err)
}
var noSignatureSignedManifest schema1.SignedManifest
noSignatureSignedManifest.Raw = noSignatureManifestBytes
err = json.Unmarshal(noSignatureManifestBytes, &noSignatureSignedManifest.Manifest)
if err != nil {
t.Fatal("error unmarshaling manifest:", err)
}
verifiedManifest, err = verifyManifest(&noSignatureSignedManifest, expectedWholeFileDigest)
if err != nil {
t.Fatal("validateManifest failed:", err)
}
if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
t.Fatal("unexpected FSLayer in no-signature manifest")
}
}

126
graph/push.go
View File

@ -1,126 +0,0 @@
package graph
import (
"fmt"
"io"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/cliconfig"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/registry"
)
// ImagePushConfig stores push configuration.
type ImagePushConfig struct {
// MetaHeaders store HTTP headers with metadata about the image
// (DockerHeaders with prefix X-Meta- in the request).
MetaHeaders map[string][]string
// AuthConfig holds authentication credentials for authenticating with
// the registry.
AuthConfig *cliconfig.AuthConfig
// Tag is the specific variant of the image to be pushed.
// If no tag is provided, all tags will be pushed.
Tag string
// OutStream is the output writer for showing the status of the push
// operation.
OutStream io.Writer
}
// pusher is an interface that abstracts pushing for different API versions.
type pusher interface {
// Push tries to push the image configured at the creation of Pusher.
// Push returns an error if any, as well as a boolean that determines whether to retry Push on the next configured endpoint.
//
// TODO(tiborvass): have Push() take a reference to repository + tag, so that the pusher itself is repository-agnostic.
Push() (fallback bool, err error)
}
// newPusher creates a new Pusher interface that will push to either a v1 or v2
// registry. The endpoint argument contains a Version field that determines
// whether a v1 or v2 pusher will be created. The other parameters are passed
// through to the underlying pusher implementation for use during the actual
// push operation.
func (s *TagStore) newPusher(endpoint registry.APIEndpoint, localRepo repository, repoInfo *registry.RepositoryInfo, imagePushConfig *ImagePushConfig, sf *streamformatter.StreamFormatter) (pusher, error) {
switch endpoint.Version {
case registry.APIVersion2:
return &v2Pusher{
TagStore: s,
endpoint: endpoint,
localRepo: localRepo,
repoInfo: repoInfo,
config: imagePushConfig,
sf: sf,
layersPushed: make(map[digest.Digest]bool),
}, nil
case registry.APIVersion1:
return &v1Pusher{
TagStore: s,
endpoint: endpoint,
localRepo: localRepo,
repoInfo: repoInfo,
config: imagePushConfig,
sf: sf,
}, nil
}
return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
}
// Push initiates a push operation on the repository named localName.
func (s *TagStore) Push(localName string, imagePushConfig *ImagePushConfig) error {
// FIXME: Allow to interrupt current push when new push of same image is done.
var sf = streamformatter.NewJSONStreamFormatter()
// Resolve the Repository name from fqn to RepositoryInfo
repoInfo, err := s.registryService.ResolveRepository(localName)
if err != nil {
return err
}
endpoints, err := s.registryService.LookupPushEndpoints(repoInfo.CanonicalName)
if err != nil {
return err
}
reposLen := 1
if imagePushConfig.Tag == "" {
reposLen = len(s.Repositories[repoInfo.LocalName])
}
imagePushConfig.OutStream.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen))
// If it fails, try to get the repository
localRepo, exists := s.Repositories[repoInfo.LocalName]
if !exists {
return fmt.Errorf("Repository does not exist: %s", repoInfo.LocalName)
}
var lastErr error
for _, endpoint := range endpoints {
logrus.Debugf("Trying to push %s to %s %s", repoInfo.CanonicalName, endpoint.URL, endpoint.Version)
pusher, err := s.newPusher(endpoint, localRepo, repoInfo, imagePushConfig, sf)
if err != nil {
lastErr = err
continue
}
if fallback, err := pusher.Push(); err != nil {
if fallback {
lastErr = err
continue
}
logrus.Debugf("Not continuing with error: %v", err)
return err
}
s.eventsService.Log("push", repoInfo.LocalName, "")
return nil
}
if lastErr == nil {
lastErr = fmt.Errorf("no endpoints found for %s", repoInfo.CanonicalName)
}
return lastErr
}

354
graph/push_v1.go
View File

@ -1,354 +0,0 @@
package graph
import (
"fmt"
"io"
"os"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
)
type v1Pusher struct {
*TagStore
endpoint registry.APIEndpoint
localRepo repository
repoInfo *registry.RepositoryInfo
config *ImagePushConfig
sf *streamformatter.StreamFormatter
session *registry.Session
out io.Writer
}
func (p *v1Pusher) Push() (fallback bool, err error) {
tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name)
if err != nil {
return false, err
}
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
tr := transport.NewTransport(
// TODO(tiborvass): was NoTimeout
registry.NewTransport(tlsConfig),
registry.DockerHeaders(p.config.MetaHeaders)...,
)
client := registry.HTTPClient(tr)
v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
if err != nil {
logrus.Debugf("Could not get v1 endpoint: %v", err)
return true, err
}
p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
if err != nil {
// TODO(dmcgowan): Check if should fallback
return true, err
}
if err := p.pushRepository(p.config.Tag); err != nil {
// TODO(dmcgowan): Check if should fallback
return false, err
}
return false, nil
}
// Retrieve the all the images to be uploaded in the correct order
func (p *v1Pusher) getImageList(requestedTag string) ([]string, map[string][]string, error) {
var (
imageList []string
imagesSeen = make(map[string]bool)
tagsByImage = make(map[string][]string)
)
for tag, id := range p.localRepo {
if requestedTag != "" && requestedTag != tag {
// Include only the requested tag.
continue
}
if utils.DigestReference(tag) {
// Ignore digest references.
continue
}
var imageListForThisTag []string
tagsByImage[id] = append(tagsByImage[id], tag)
for img, err := p.graph.Get(id); img != nil; img, err = p.graph.GetParent(img) {
if err != nil {
return nil, nil, err
}
if imagesSeen[img.ID] {
// This image is already on the list, we can ignore it and all its parents
break
}
imagesSeen[img.ID] = true
imageListForThisTag = append(imageListForThisTag, img.ID)
}
// reverse the image list for this tag (so the "most"-parent image is first)
for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 {
imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i]
}
// append to main image list
imageList = append(imageList, imageListForThisTag...)
}
if len(imageList) == 0 {
return nil, nil, fmt.Errorf("No images found for the requested repository / tag")
}
logrus.Debugf("Image list: %v", imageList)
logrus.Debugf("Tags by image: %v", tagsByImage)
return imageList, tagsByImage, nil
}
// createImageIndex returns an index of an image's layer IDs and tags.
func (s *TagStore) createImageIndex(images []string, tags map[string][]string) []*registry.ImgData {
var imageIndex []*registry.ImgData
for _, id := range images {
if tags, hasTags := tags[id]; hasTags {
// If an image has tags you must add an entry in the image index
// for each tag
for _, tag := range tags {
imageIndex = append(imageIndex, &registry.ImgData{
ID: id,
Tag: tag,
})
}
continue
}
// If the image does not have a tag it still needs to be sent to the
// registry with an empty tag so that it is associated with the repository
imageIndex = append(imageIndex, &registry.ImgData{
ID: id,
Tag: "",
})
}
return imageIndex
}
type imagePushData struct {
id string
compatibilityID string
endpoint string
}
// lookupImageOnEndpoint checks the specified endpoint to see if an image exists
// and if it is absent then it sends the image id to the channel to be pushed.
func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) {
defer wg.Done()
for image := range images {
if err := p.session.LookupRemoteImage(image.compatibilityID, image.endpoint); err != nil {
logrus.Errorf("Error in LookupRemoteImage: %s", err)
imagesToPush <- image.id
continue
}
p.out.Write(p.sf.FormatStatus("", "Image %s already pushed, skipping", stringid.TruncateID(image.id)))
}
}
func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags map[string][]string, repo *registry.RepositoryData) error {
workerCount := len(imageIDs)
// start a maximum of 5 workers to check if images exist on the specified endpoint.
if workerCount > 5 {
workerCount = 5
}
var (
wg = &sync.WaitGroup{}
imageData = make(chan imagePushData, workerCount*2)
imagesToPush = make(chan string, workerCount*2)
pushes = make(chan map[string]struct{}, 1)
)
for i := 0; i < workerCount; i++ {
wg.Add(1)
go p.lookupImageOnEndpoint(wg, imageData, imagesToPush)
}
// start a go routine that consumes the images to push
go func() {
shouldPush := make(map[string]struct{})
for id := range imagesToPush {
shouldPush[id] = struct{}{}
}
pushes <- shouldPush
}()
for _, id := range imageIDs {
compatibilityID, err := p.getV1ID(id)
if err != nil {
return err
}
imageData <- imagePushData{
id: id,
compatibilityID: compatibilityID,
endpoint: endpoint,
}
}
// close the channel to notify the workers that there will be no more images to check.
close(imageData)
wg.Wait()
close(imagesToPush)
// wait for all the images that require pushes to be collected into a consumable map.
shouldPush := <-pushes
// finish by pushing any images and tags to the endpoint. The order that the images are pushed
// is very important that is why we are still iterating over the ordered list of imageIDs.
for _, id := range imageIDs {
if _, push := shouldPush[id]; push {
if _, err := p.pushImage(id, endpoint); err != nil {
// FIXME: Continue on error?
return err
}
}
for _, tag := range tags[id] {
p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag))
compatibilityID, err := p.getV1ID(id)
if err != nil {
return err
}
if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, compatibilityID, tag, endpoint); err != nil {
return err
}
}
}
return nil
}
// pushRepository pushes layers that do not already exist on the registry.
func (p *v1Pusher) pushRepository(tag string) error {
logrus.Debugf("Local repo: %s", p.localRepo)
p.out = ioutils.NewWriteFlusher(p.config.OutStream)
imgList, tags, err := p.getImageList(tag)
if err != nil {
return err
}
p.out.Write(p.sf.FormatStatus("", "Sending image list"))
imageIndex := p.createImageIndex(imgList, tags)
logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo)
for _, data := range imageIndex {
logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
// convert IDs to compatibilityIDs, imageIndex only used in registry calls
data.ID, err = p.getV1ID(data.ID)
if err != nil {
return err
}
}
if _, found := p.poolAdd("push", p.repoInfo.LocalName); found {
return fmt.Errorf("push or pull %s is already in progress", p.repoInfo.LocalName)
}
defer p.poolRemove("push", p.repoInfo.LocalName)
// Register all the images in a repository with the registry
// If an image is not in this list it will not be associated with the repository
repoData, err := p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, false, nil)
if err != nil {
return err
}
nTag := 1
if tag == "" {
nTag = len(p.localRepo)
}
p.out.Write(p.sf.FormatStatus("", "Pushing repository %s (%d tags)", p.repoInfo.CanonicalName, nTag))
// push the repository to each of the endpoints only if it does not exist.
for _, endpoint := range repoData.Endpoints {
if err := p.pushImageToEndpoint(endpoint, imgList, tags, repoData); err != nil {
return err
}
}
_, err = p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, true, repoData.Endpoints)
return err
}
func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) {
jsonRaw, err := p.getV1Config(imgID)
if err != nil {
return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err)
}
p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil))
compatibilityID, err := p.getV1ID(imgID)
if err != nil {
return "", err
}
// General rule is to use ID for graph accesses and compatibilityID for
// calls to session.registry()
imgData := &registry.ImgData{
ID: compatibilityID,
}
// Send the json
if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
if err == registry.ErrAlreadyExists {
p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image already pushed, skipping", nil))
return "", nil
}
return "", err
}
layerData, err := p.graph.tempLayerArchive(imgID, p.sf, p.out)
if err != nil {
return "", fmt.Errorf("Failed to generate layer archive: %s", err)
}
defer os.RemoveAll(layerData.Name())
// Send the layer
logrus.Debugf("rendered layer for %s of [%d] size", imgID, layerData.Size)
checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID,
progressreader.New(progressreader.Config{
In: layerData,
Out: p.out,
Formatter: p.sf,
Size: layerData.Size,
NewLines: false,
ID: stringid.TruncateID(imgID),
Action: "Pushing",
}), ep, jsonRaw)
if err != nil {
return "", err
}
imgData.Checksum = checksum
imgData.ChecksumPayload = checksumPayload
// Send the checksum
if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil {
return "", err
}
p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image successfully pushed", nil))
return imgData.Checksum, nil
}
// getV1ID returns the compatibilityID for the ID in the graph. compatibilityID
// is read from from the v1Compatibility config file in the disk.
func (p *v1Pusher) getV1ID(id string) (string, error) {
jsonData, err := p.getV1Config(id)
if err != nil {
return "", err
}
img, err := image.NewImgJSON(jsonData)
if err != nil {
return "", err
}
return img.ID, nil
}
// getV1Config returns v1Compatibility config for the image in the graph. If
// there is no v1Compatibility file on disk for the image
func (p *v1Pusher) getV1Config(id string) ([]byte, error) {
jsonData, err := p.graph.generateV1CompatibilityChain(id)
if err != nil {
return nil, err
}
return jsonData, nil
}

397
graph/push_v2.go
View File

@ -1,397 +0,0 @@
package graph
import (
"bufio"
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/progressreader"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/runconfig"
"github.com/docker/docker/utils"
"golang.org/x/net/context"
)
const compressionBufSize = 32768
type v2Pusher struct {
*TagStore
endpoint registry.APIEndpoint
localRepo repository
repoInfo *registry.RepositoryInfo
config *ImagePushConfig
sf *streamformatter.StreamFormatter
repo distribution.Repository
// layersPushed is the set of layers known to exist on the remote side.
// This avoids redundant queries when pushing multiple tags that
// involve the same layers.
layersPushed map[digest.Digest]bool
}
func (p *v2Pusher) Push() (fallback bool, err error) {
p.repo, err = newV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull")
if err != nil {
logrus.Debugf("Error getting v2 registry: %v", err)
return true, err
}
return false, p.pushV2Repository(p.config.Tag)
}
func (p *v2Pusher) getImageTags(askedTag string) ([]string, error) {
logrus.Debugf("Checking %q against %#v", askedTag, p.localRepo)
if len(askedTag) > 0 {
if _, ok := p.localRepo[askedTag]; !ok || utils.DigestReference(askedTag) {
return nil, fmt.Errorf("Tag does not exist for %s", askedTag)
}
return []string{askedTag}, nil
}
var tags []string
for tag := range p.localRepo {
if !utils.DigestReference(tag) {
tags = append(tags, tag)
}
}
return tags, nil
}
func (p *v2Pusher) pushV2Repository(tag string) error {
localName := p.repoInfo.LocalName
if _, found := p.poolAdd("push", localName); found {
return fmt.Errorf("push or pull %s is already in progress", localName)
}
defer p.poolRemove("push", localName)
tags, err := p.getImageTags(tag)
if err != nil {
return fmt.Errorf("error getting tags for %s: %s", localName, err)
}
if len(tags) == 0 {
return fmt.Errorf("no tags to push for %s", localName)
}
for _, tag := range tags {
if err := p.pushV2Tag(tag); err != nil {
return err
}
}
return nil
}
func (p *v2Pusher) pushV2Tag(tag string) error {
logrus.Debugf("Pushing repository: %s:%s", p.repo.Name(), tag)
layerID, exists := p.localRepo[tag]
if !exists {
return fmt.Errorf("tag does not exist: %s", tag)
}
layersSeen := make(map[string]bool)
layer, err := p.graph.Get(layerID)
if err != nil {
return err
}
m := &schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: p.repo.Name(),
Tag: tag,
Architecture: layer.Architecture,
FSLayers: []schema1.FSLayer{},
History: []schema1.History{},
}
var metadata runconfig.Config
if layer != nil && layer.Config != nil {
metadata = *layer.Config
}
out := p.config.OutStream
for ; layer != nil; layer, err = p.graph.GetParent(layer) {
if err != nil {
return err
}
// break early if layer has already been seen in this image,
// this prevents infinite loops on layers which loopback, this
// cannot be prevented since layer IDs are not merkle hashes
// TODO(dmcgowan): throw error if no valid use case is found
if layersSeen[layer.ID] {
break
}
// Skip the base layer on Windows. This cannot be pushed.
if allowBaseParentImage && layer.Parent == "" {
break
}
logrus.Debugf("Pushing layer: %s", layer.ID)
if layer.Config != nil && metadata.Image != layer.ID {
if err := runconfig.Merge(&metadata, layer.Config); err != nil {
return err
}
}
var exists bool
dgst, err := p.graph.getLayerDigestWithLock(layer.ID)
switch err {
case nil:
if p.layersPushed[dgst] {
exists = true
// break out of switch, it is already known that
// the push is not needed and therefore doing a
// stat is unnecessary
break
}
_, err := p.repo.Blobs(context.Background()).Stat(context.Background(), dgst)
switch err {
case nil:
exists = true
out.Write(p.sf.FormatProgress(stringid.TruncateID(layer.ID), "Image already exists", nil))
case distribution.ErrBlobUnknown:
// nop
default:
out.Write(p.sf.FormatProgress(stringid.TruncateID(layer.ID), "Image push failed", nil))
return err
}
case errDigestNotSet:
// nop
case digest.ErrDigestInvalidFormat, digest.ErrDigestUnsupported:
return fmt.Errorf("error getting image checksum: %v", err)
}
// if digest was empty or not saved, or if blob does not exist on the remote repository,
// then fetch it.
if !exists {
var pushDigest digest.Digest
if pushDigest, err = p.pushV2Image(p.repo.Blobs(context.Background()), layer); err != nil {
return err
}
if dgst == "" {
// Cache new checksum
if err := p.graph.setLayerDigestWithLock(layer.ID, pushDigest); err != nil {
return err
}
}
dgst = pushDigest
}
// read v1Compatibility config, generate new if needed
jsonData, err := p.graph.generateV1CompatibilityChain(layer.ID)
if err != nil {
return err
}
m.FSLayers = append(m.FSLayers, schema1.FSLayer{BlobSum: dgst})
m.History = append(m.History, schema1.History{V1Compatibility: string(jsonData)})
layersSeen[layer.ID] = true
p.layersPushed[dgst] = true
}
// Fix parent chain if necessary
if err = fixHistory(m); err != nil {
return err
}
logrus.Infof("Signed manifest for %s:%s using daemon's key: %s", p.repo.Name(), tag, p.trustKey.KeyID())
signed, err := schema1.Sign(m, p.trustKey)
if err != nil {
return err
}
manifestDigest, manifestSize, err := digestFromManifest(signed, p.repo.Name())
if err != nil {
return err
}
if manifestDigest != "" {
out.Write(p.sf.FormatStatus("", "%s: digest: %s size: %d", tag, manifestDigest, manifestSize))
}
manSvc, err := p.repo.Manifests(context.Background())
if err != nil {
return err
}
return manSvc.Put(signed)
}
// fixHistory makes sure that the manifest has parent IDs that are consistent
// with its image IDs. Because local image IDs are generated from the
// configuration and filesystem contents, but IDs in the manifest are preserved
// from the original pull, it's possible to have inconsistencies where parent
// IDs don't match up with the other IDs in the manifest. This happens in the
// case where an engine pulls images where are identical except the IDs from the
// manifest - the local ID will be the same, and one of the v1Compatibility
// files gets discarded.
func fixHistory(m *schema1.Manifest) error {
var lastID string
for i := len(m.History) - 1; i >= 0; i-- {
var historyEntry map[string]*json.RawMessage
if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), &historyEntry); err != nil {
return err
}
idJSON, present := historyEntry["id"]
if !present || idJSON == nil {
return errors.New("missing id key in v1compatibility file")
}
var id string
if err := json.Unmarshal(*idJSON, &id); err != nil {
return err
}
parentJSON, present := historyEntry["parent"]
if i == len(m.History)-1 {
// The base layer must not reference a parent layer,
// otherwise the manifest is incomplete. There is an
// exception for Windows to handle base layers.
if !allowBaseParentImage && present && parentJSON != nil {
var parent string
if err := json.Unmarshal(*parentJSON, &parent); err != nil {
return err
}
if parent != "" {
logrus.Debugf("parent id mismatch detected; fixing. parent reference: %s", parent)
delete(historyEntry, "parent")
fixedHistory, err := json.Marshal(historyEntry)
if err != nil {
return err
}
m.History[i].V1Compatibility = string(fixedHistory)
}
}
} else {
// For all other layers, the parent ID should equal the
// ID of the next item in the history list. If it
// doesn't, fix it up (but preserve all other fields,
// possibly including fields that aren't known to this
// engine version).
if !present || parentJSON == nil {
return errors.New("missing parent key in v1compatibility file")
}
var parent string
if err := json.Unmarshal(*parentJSON, &parent); err != nil {
return err
}
if parent != lastID {
logrus.Debugf("parent id mismatch detected; fixing. parent reference: %s actual id: %s", parent, id)
historyEntry["parent"] = rawJSON(lastID)
fixedHistory, err := json.Marshal(historyEntry)
if err != nil {
return err
}
m.History[i].V1Compatibility = string(fixedHistory)
}
}
lastID = id
}
return nil
}
func rawJSON(value interface{}) *json.RawMessage {
jsonval, err := json.Marshal(value)
if err != nil {
return nil
}
return (*json.RawMessage)(&jsonval)
}
func (p *v2Pusher) pushV2Image(bs distribution.BlobService, img *image.Image) (digest.Digest, error) {
out := p.config.OutStream
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Preparing", nil))
image, err := p.graph.Get(img.ID)
if err != nil {
return "", err
}
arch, err := p.graph.tarLayer(image)
if err != nil {
return "", err
}
defer arch.Close()
// Send the layer
layerUpload, err := bs.Create(context.Background())
if err != nil {
return "", err
}
defer layerUpload.Close()
reader := progressreader.New(progressreader.Config{
In: ioutil.NopCloser(arch), // we'll take care of close here.
Out: out,
Formatter: p.sf,
// TODO(stevvooe): This may cause a size reporting error. Try to get
// this from tar-split or elsewhere. The main issue here is that we
// don't want to buffer to disk *just* to calculate the size.
Size: img.Size,
NewLines: false,
ID: stringid.TruncateID(img.ID),
Action: "Pushing",
})
digester := digest.Canonical.New()
// HACK: The MultiWriter doesn't write directly to layerUpload because
// we must make sure the ReadFrom is used, not Write. Using Write would
// send a PATCH request for every Write call.
pipeReader, pipeWriter := io.Pipe()
// Use a bufio.Writer to avoid excessive chunking in HTTP request.
bufWriter := bufio.NewWriterSize(io.MultiWriter(pipeWriter, digester.Hash()), compressionBufSize)
compressor := gzip.NewWriter(bufWriter)
go func() {
_, err := io.Copy(compressor, reader)
if err == nil {
err = compressor.Close()
}
if err == nil {
err = bufWriter.Flush()
}
if err != nil {
pipeWriter.CloseWithError(err)
} else {
pipeWriter.Close()
}
}()
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pushing", nil))
nn, err := layerUpload.ReadFrom(pipeReader)
pipeReader.Close()
if err != nil {
return "", err
}
dgst := digester.Digest()
if _, err := layerUpload.Commit(context.Background(), distribution.Descriptor{Digest: dgst}); err != nil {
return "", err
}
logrus.Debugf("uploaded layer %s (%s), %d bytes", img.ID, dgst, nn)
out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pushed", nil))
return dgst, nil
}

116
graph/registry.go
View File

@ -1,116 +0,0 @@
package graph
import (
"errors"
"net"
"net/http"
"net/url"
"time"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/registry/client"
"github.com/docker/distribution/registry/client/auth"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/cliconfig"
"github.com/docker/docker/registry"
"golang.org/x/net/context"
)
type dumbCredentialStore struct {
auth *cliconfig.AuthConfig
}
func (dcs dumbCredentialStore) Basic(*url.URL) (string, string) {
return dcs.auth.Username, dcs.auth.Password
}
// newV2Repository returns a repository (v2 only). It creates a HTTP transport
// providing timeout settings and authentication support, and also verifies the
// remote API version.
func newV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *cliconfig.AuthConfig, actions ...string) (distribution.Repository, error) {
ctx := context.Background()
repoName := repoInfo.CanonicalName
// If endpoint does not support CanonicalName, use the RemoteName instead
if endpoint.TrimHostname {
repoName = repoInfo.RemoteName
}
// TODO(dmcgowan): Call close idle connections when complete, use keep alive
base := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: endpoint.TLSConfig,
// TODO(dmcgowan): Call close idle connections when complete and use keep alive
DisableKeepAlives: true,
}
modifiers := registry.DockerHeaders(metaHeaders)
authTransport := transport.NewTransport(base, modifiers...)
pingClient := &http.Client{
Transport: authTransport,
Timeout: 15 * time.Second,
}
endpointStr := strings.TrimRight(endpoint.URL, "/") + "/v2/"
req, err := http.NewRequest("GET", endpointStr, nil)
if err != nil {
return nil, err
}
resp, err := pingClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
versions := auth.APIVersions(resp, endpoint.VersionHeader)
if endpoint.VersionHeader != "" && len(endpoint.Versions) > 0 {
var foundVersion bool
for _, version := range endpoint.Versions {
for _, pingVersion := range versions {
if version == pingVersion {
foundVersion = true
}
}
}
if !foundVersion {
return nil, errors.New("endpoint does not support v2 API")
}
}
challengeManager := auth.NewSimpleChallengeManager()
if err := challengeManager.AddResponse(resp); err != nil {
return nil, err
}
creds := dumbCredentialStore{auth: authConfig}
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, actions...)
basicHandler := auth.NewBasicHandler(creds)
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
tr := transport.NewTransport(base, modifiers...)
return client.NewRepository(ctx, repoName, endpoint.URL, tr)
}
func digestFromManifest(m *schema1.SignedManifest, localName string) (digest.Digest, int, error) {
payload, err := m.Payload()
if err != nil {
// If this failed, the signatures section was corrupted
// or missing. Treat the entire manifest as the payload.
payload = m.Raw
}
manifestDigest, err := digest.FromBytes(payload)
if err != nil {
logrus.Infof("Could not compute manifest digest for %s:%s : %v", localName, m.Tag, err)
}
return manifestDigest, len(payload), nil
}

89
graph/service.go
View File

@ -1,89 +0,0 @@
package graph
import (
"fmt"
"io"
"runtime"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/types"
"github.com/docker/docker/utils"
)
// Lookup looks up an image by name in a TagStore and returns it as an
// ImageInspect structure.
func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) {
image, err := s.LookupImage(name)
if err != nil || image == nil {
return nil, fmt.Errorf("No such image: %s", name)
}
var repoTags = make([]string, 0)
var repoDigests = make([]string, 0)
s.Lock()
for repoName, repository := range s.Repositories {
for ref, id := range repository {
if id == image.ID {
imgRef := utils.ImageReference(repoName, ref)
if utils.DigestReference(ref) {
repoDigests = append(repoDigests, imgRef)
} else {
repoTags = append(repoTags, imgRef)
}
}
}
}
s.Unlock()
imageInspect := &types.ImageInspect{
ID: image.ID,
RepoTags: repoTags,
RepoDigests: repoDigests,
Parent: image.Parent,
Comment: image.Comment,
Created: image.Created.Format(time.RFC3339Nano),
Container: image.Container,
ContainerConfig: &image.ContainerConfig,
DockerVersion: image.DockerVersion,
Author: image.Author,
Config: image.Config,
Architecture: image.Architecture,
Os: image.OS,
Size: image.Size,
VirtualSize: s.graph.getParentsSize(image) + image.Size,
}
imageInspect.GraphDriver.Name = s.graph.driver.String()
graphDriverData, err := s.graph.driver.GetMetadata(image.ID)
if err != nil {
return nil, err
}
imageInspect.GraphDriver.Data = graphDriverData
return imageInspect, nil
}
// imageTarLayer return the tarLayer of the image
func (s *TagStore) imageTarLayer(name string, dest io.Writer) error {
if image, err := s.LookupImage(name); err == nil && image != nil {
// On Windows, the base layer cannot be exported
if runtime.GOOS != "windows" || image.Parent != "" {
fs, err := s.graph.tarLayer(image)
if err != nil {
return err
}
defer fs.Close()
written, err := io.Copy(dest, fs)
if err != nil {
return err
}
logrus.Debugf("rendered layer for %s of [%d] size", image.ID, written)
}
return nil
}
return fmt.Errorf("No such image: %s", name)
}

431
graph/tags.go
View File

@ -1,431 +0,0 @@
package graph
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/docker/distribution/digest"
"github.com/docker/docker/daemon/events"
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/broadcaster"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
"github.com/docker/libtrust"
)
// ErrNameIsNotExist returned when there is no image with requested name.
var ErrNameIsNotExist = errors.New("image with specified name does not exist")
// TagStore manages repositories. It encompasses the Graph used for versioned
// storage, as well as various services involved in pushing and pulling
// repositories.
type TagStore struct {
path string
graph *Graph
// Repositories is a map of repositories, indexed by name.
Repositories map[string]repository
trustKey libtrust.PrivateKey
sync.Mutex
// FIXME: move push/pull-related fields
// to a helper type
pullingPool map[string]*broadcaster.Buffered
pushingPool map[string]*broadcaster.Buffered
registryService *registry.Service
eventsService *events.Events
}
// repository maps tags to image IDs.
type repository map[string]string
// TagStoreConfig provides parameters for a new TagStore.
type TagStoreConfig struct {
// Graph is the versioned image store
Graph *Graph
// Key is the private key to use for signing manifests.
Key libtrust.PrivateKey
// Registry is the registry service to use for TLS configuration and
// endpoint lookup.
Registry *registry.Service
// Events is the events service to use for logging.
Events *events.Events
}
// NewTagStore creates a new TagStore at specified path, using the parameters
// and services provided in cfg.
func NewTagStore(path string, cfg *TagStoreConfig) (*TagStore, error) {
abspath, err := filepath.Abs(path)
if err != nil {
return nil, err
}
store := &TagStore{
path: abspath,
graph: cfg.Graph,
trustKey: cfg.Key,
Repositories: make(map[string]repository),
pullingPool: make(map[string]*broadcaster.Buffered),
pushingPool: make(map[string]*broadcaster.Buffered),
registryService: cfg.Registry,
eventsService: cfg.Events,
}
// Load the json file if it exists, otherwise create it.
if err := store.reload(); os.IsNotExist(err) {
if err := store.save(); err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return store, nil
}
func (store *TagStore) save() error {
// Store the json ball
jsonData, err := json.Marshal(store)
if err != nil {
return err
}
if err := ioutil.WriteFile(store.path, jsonData, 0600); err != nil {
return err
}
return nil
}
func (store *TagStore) reload() error {
f, err := os.Open(store.path)
if err != nil {
return err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&store); err != nil {
return err
}
return nil
}
// LookupImage returns pointer to an Image struct corresponding to the given
// name. The name can include an optional tag; otherwise the default tag will
// be used.
func (store *TagStore) LookupImage(name string) (*image.Image, error) {
repoName, ref := parsers.ParseRepositoryTag(name)
if ref == "" {
ref = tags.DefaultTag
}
var (
err error
img *image.Image
)
img, err = store.getImage(repoName, ref)
if err != nil {
return nil, err
}
if img != nil {
return img, nil
}
// name must be an image ID.
store.Lock()
defer store.Unlock()
if img, err = store.graph.Get(name); err != nil {
return nil, err
}
return img, nil
}
// GetID returns ID for image name.
func (store *TagStore) GetID(name string) (string, error) {
repoName, ref := parsers.ParseRepositoryTag(name)
if ref == "" {
ref = tags.DefaultTag
}
store.Lock()
defer store.Unlock()
repoName = registry.NormalizeLocalName(repoName)
repo, ok := store.Repositories[repoName]
if !ok {
return "", ErrNameIsNotExist
}
id, ok := repo[ref]
if !ok {
return "", ErrNameIsNotExist
}
return id, nil
}
// ByID returns a reverse-lookup table of all the names which refer to each
// image - e.g. {"43b5f19b10584": {"base:latest", "base:v1"}}
func (store *TagStore) ByID() map[string][]string {
store.Lock()
defer store.Unlock()
byID := make(map[string][]string)
for repoName, repository := range store.Repositories {
for tag, id := range repository {
name := utils.ImageReference(repoName, tag)
if _, exists := byID[id]; !exists {
byID[id] = []string{name}
} else {
byID[id] = append(byID[id], name)
sort.Strings(byID[id])
}
}
}
return byID
}
// HasReferences returns whether or not the given image is referenced in one or
// more repositories.
func (store *TagStore) HasReferences(img *image.Image) bool {
return len(store.ByID()[img.ID]) > 0
}
// Delete deletes a repository or a specific tag. If ref is empty, the entire
// repository named repoName will be deleted; otherwise only the tag named by
// ref will be deleted.
func (store *TagStore) Delete(repoName, ref string) (bool, error) {
store.Lock()
defer store.Unlock()
deleted := false
if err := store.reload(); err != nil {
return false, err
}
if ref == "" {
// Delete the whole repository.
delete(store.Repositories, repoName)
return true, store.save()
}
repoRefs, exists := store.Repositories[repoName]
if !exists {
return false, fmt.Errorf("No such repository: %s", repoName)
}
if _, exists := repoRefs[ref]; exists {
delete(repoRefs, ref)
if len(repoRefs) == 0 {
delete(store.Repositories, repoName)
}
deleted = true
}
return deleted, store.save()
}
// Tag creates a tag in the repository reponame, pointing to the image named
// imageName. If force is true, an existing tag with the same name may be
// overwritten.
func (store *TagStore) Tag(repoName, tag, imageName string, force bool) error {
return store.setLoad(repoName, tag, imageName, force, nil)
}
// setLoad stores the image to the store.
// If the imageName is already in the repo then a '-f' flag should be used to replace existing image.
func (store *TagStore) setLoad(repoName, tag, imageName string, force bool, out io.Writer) error {
img, err := store.LookupImage(imageName)
store.Lock()
defer store.Unlock()
if err != nil {
return err
}
if tag == "" {
tag = tags.DefaultTag
}
if err := validateRepoName(repoName); err != nil {
return err
}
if err := tags.ValidateTagName(tag); err != nil {
return err
}
if err := store.reload(); err != nil {
return err
}
var repo repository
repoName = registry.NormalizeLocalName(repoName)
if r, exists := store.Repositories[repoName]; exists {
repo = r
if old, exists := store.Repositories[repoName][tag]; exists {
if !force {
return fmt.Errorf("Conflict: Tag %s:%s is already set to image %s, if you want to replace it, please use -f option", repoName, tag, old[:12])
}
if old != img.ID && out != nil {
fmt.Fprintf(out, "The image %s:%s already exists, renaming the old one with ID %s to empty string\n", repoName, tag, old[:12])
}
}
} else {
repo = make(map[string]string)
store.Repositories[repoName] = repo
}
repo[tag] = img.ID
return store.save()
}
// setDigest creates a digest reference to an image ID.
func (store *TagStore) setDigest(repoName, digest, imageName string) error {
img, err := store.LookupImage(imageName)
if err != nil {
return err
}
if err := validateRepoName(repoName); err != nil {
return err
}
if err := validateDigest(digest); err != nil {
return err
}
store.Lock()
defer store.Unlock()
if err := store.reload(); err != nil {
return err
}
repoName = registry.NormalizeLocalName(repoName)
repoRefs, exists := store.Repositories[repoName]
if !exists {
repoRefs = repository{}
store.Repositories[repoName] = repoRefs
} else if oldID, exists := repoRefs[digest]; exists && oldID != img.ID {
return fmt.Errorf("Conflict: Digest %s is already set to image %s", digest, oldID)
}
repoRefs[digest] = img.ID
return store.save()
}
// get returns the repository tag/image map for a given repository.
func (store *TagStore) get(repoName string) (repository, error) {
store.Lock()
defer store.Unlock()
if err := store.reload(); err != nil {
return nil, err
}
repoName = registry.NormalizeLocalName(repoName)
if r, exists := store.Repositories[repoName]; exists {
return r, nil
}
return nil, nil
}
// getImage returns a pointer to an Image structure describing the image
// referred to by refOrID inside repository repoName.
func (store *TagStore) getImage(repoName, refOrID string) (*image.Image, error) {
repo, err := store.get(repoName)
if err != nil {
return nil, err
}
if repo == nil {
return nil, nil
}
store.Lock()
defer store.Unlock()
if imgID, exists := repo[refOrID]; exists {
return store.graph.Get(imgID)
}
// If no matching tag is found, search through images for a matching image id
// iff it looks like a short ID or would look like a short ID
if stringid.IsShortID(stringid.TruncateID(refOrID)) {
for _, revision := range repo {
if strings.HasPrefix(revision, refOrID) {
return store.graph.Get(revision)
}
}
}
return nil, nil
}
// validateRepoName validates the name of a repository.
func validateRepoName(name string) error {
if name == "" {
return fmt.Errorf("Repository name can't be empty")
}
if name == "scratch" {
return fmt.Errorf("'scratch' is a reserved name")
}
return nil
}
func validateDigest(dgst string) error {
if dgst == "" {
return errors.New("digest can't be empty")
}
if _, err := digest.ParseDigest(dgst); err != nil {
return err
}
return nil
}
// poolAdd checks if a push or pull is already running, and returns
// (broadcaster, true) if a running operation is found. Otherwise, it creates a
// new one and returns (broadcaster, false).
func (store *TagStore) poolAdd(kind, key string) (*broadcaster.Buffered, bool) {
store.Lock()
defer store.Unlock()
if p, exists := store.pullingPool[key]; exists {
return p, true
}
if p, exists := store.pushingPool[key]; exists {
return p, true
}
broadcaster := broadcaster.NewBuffered()
switch kind {
case "pull":
store.pullingPool[key] = broadcaster
case "push":
store.pushingPool[key] = broadcaster
default:
panic("Unknown pool type")
}
return broadcaster, false
}
func (store *TagStore) poolRemoveWithError(kind, key string, broadcasterResult error) error {
store.Lock()
defer store.Unlock()
switch kind {
case "pull":
if broadcaster, exists := store.pullingPool[key]; exists {
broadcaster.CloseWithError(broadcasterResult)
delete(store.pullingPool, key)
}
case "push":
if broadcaster, exists := store.pushingPool[key]; exists {
broadcaster.CloseWithError(broadcasterResult)
delete(store.pushingPool, key)
}
default:
return fmt.Errorf("Unknown pool type")
}
return nil
}
func (store *TagStore) poolRemove(kind, key string) error {
return store.poolRemoveWithError(kind, key, nil)
}

36
graph/tags/tags.go
View File

@ -1,36 +0,0 @@
package tags
import (
"fmt"
"regexp"
"github.com/docker/distribution/reference"
)
// DefaultTag defines the default tag used when performing images related actions and no tag string is specified
const DefaultTag = "latest"
var anchoredTagRegexp = regexp.MustCompile(`^` + reference.TagRegexp.String() + `$`)
// ErrTagInvalidFormat is returned if tag is invalid.
type ErrTagInvalidFormat struct {
name string
}
func (e ErrTagInvalidFormat) Error() string {
return fmt.Sprintf("Illegal tag name (%s): only [A-Za-z0-9_.-] are allowed ('.' and '-' are NOT allowed in the initial), minimum 1, maximum 128 in length", e.name)
}
// ValidateTagName validates the name of a tag.
// It returns an error if the given name is an emtpy string.
// If name is not valid, it returns ErrTagInvalidFormat
func ValidateTagName(name string) error {
if name == "" {
return fmt.Errorf("tag name can't be empty")
}
if !anchoredTagRegexp.MatchString(name) {
return ErrTagInvalidFormat{name}
}
return nil
}

23
graph/tags/tags_unit_test.go
View File

@ -1,23 +0,0 @@
package tags
import (
"testing"
)
func TestValidTagName(t *testing.T) {
validTags := []string{"9", "foo", "foo-test", "bar.baz.boo"}
for _, tag := range validTags {
if err := ValidateTagName(tag); err != nil {
t.Errorf("'%s' should've been a valid tag", tag)
}
}
}
func TestInvalidTagName(t *testing.T) {
inValidTags := []string{"-9", ".foo", "-test", ".", "-"}
for _, tag := range inValidTags {
if err := ValidateTagName(tag); err == nil {
t.Errorf("'%s' should've been an invalid tag", tag)
}
}
}

205
graph/tags_unit_test.go
View File

@ -1,205 +0,0 @@
package graph
import (
"archive/tar"
"bytes"
"io"
"os"
"path"
"testing"
"github.com/docker/docker/daemon/events"
"github.com/docker/docker/daemon/graphdriver"
_ "github.com/docker/docker/daemon/graphdriver/vfs" // import the vfs driver so it is used in the tests
"github.com/docker/docker/graph/tags"
"github.com/docker/docker/image"
"github.com/docker/docker/utils"
)
const (
testOfficialImageName = "myapp"
testOfficialImageID = "1a2d3c4d4e5fa2d2a21acea242a5e2345d3aefc3e7dfa2a2a2a21a2a2ad2d234"
testOfficialImageIDShort = "1a2d3c4d4e5f"
testPrivateImageName = "127.0.0.1:8000/privateapp"
testPrivateImageID = "5bc255f8699e4ee89ac4469266c3d11515da88fdcbde45d7b069b636ff4efd81"
testPrivateImageIDShort = "5bc255f8699e"
testPrivateImageDigest = "sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb"
testPrivateImageTag = "sometag"
)
func fakeTar() (io.Reader, error) {
uid := os.Getuid()
gid := os.Getgid()
content := []byte("Hello world!\n")
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} {
hdr := new(tar.Header)
// Leaving these fields blank requires root privileges
hdr.Uid = uid
hdr.Gid = gid
hdr.Size = int64(len(content))
hdr.Name = name
if err := tw.WriteHeader(hdr); err != nil {
return nil, err
}
tw.Write([]byte(content))
}
tw.Close()
return buf, nil
}
func mkTestTagStore(root string, t *testing.T) *TagStore {
driver, err := graphdriver.New(root, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
graph, err := NewGraph(root, driver, nil, nil)
if err != nil {
t.Fatal(err)
}
tagCfg := &TagStoreConfig{
Graph: graph,
Events: events.New(),
}
store, err := NewTagStore(path.Join(root, "tags"), tagCfg)
if err != nil {
t.Fatal(err)
}
officialArchive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
img := &image.Image{ID: testOfficialImageID}
if err := graph.Register(v1Descriptor{img}, officialArchive); err != nil {
t.Fatal(err)
}
if err := store.Tag(testOfficialImageName, "", testOfficialImageID, false); err != nil {
t.Fatal(err)
}
privateArchive, err := fakeTar()
if err != nil {
t.Fatal(err)
}
img = &image.Image{ID: testPrivateImageID}
if err := graph.Register(v1Descriptor{img}, privateArchive); err != nil {
t.Fatal(err)
}
if err := store.Tag(testPrivateImageName, "", testPrivateImageID, false); err != nil {
t.Fatal(err)
}
if err := store.setDigest(testPrivateImageName, testPrivateImageDigest, testPrivateImageID); err != nil {
t.Fatal(err)
}
return store
}
func TestLookupImage(t *testing.T) {
tmp, err := utils.TestDirectory("")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
store := mkTestTagStore(tmp, t)
defer store.graph.driver.Cleanup()
officialLookups := []string{
testOfficialImageID,
testOfficialImageIDShort,
testOfficialImageName + ":" + testOfficialImageID,
testOfficialImageName + ":" + testOfficialImageIDShort,
testOfficialImageName,
testOfficialImageName + ":" + tags.DefaultTag,
"docker.io/" + testOfficialImageName,
"docker.io/" + testOfficialImageName + ":" + tags.DefaultTag,
"index.docker.io/" + testOfficialImageName,
"index.docker.io/" + testOfficialImageName + ":" + tags.DefaultTag,
"library/" + testOfficialImageName,
"library/" + testOfficialImageName + ":" + tags.DefaultTag,
"docker.io/library/" + testOfficialImageName,
"docker.io/library/" + testOfficialImageName + ":" + tags.DefaultTag,
"index.docker.io/library/" + testOfficialImageName,
"index.docker.io/library/" + testOfficialImageName + ":" + tags.DefaultTag,
}
privateLookups := []string{
testPrivateImageID,
testPrivateImageIDShort,
testPrivateImageName + ":" + testPrivateImageID,
testPrivateImageName + ":" + testPrivateImageIDShort,
testPrivateImageName,
testPrivateImageName + ":" + tags.DefaultTag,
}
invalidLookups := []string{
testOfficialImageName + ":" + "fail",
"fail:fail",
}
digestLookups := []string{
testPrivateImageName + "@" + testPrivateImageDigest,
}
for _, name := range officialLookups {
if img, err := store.LookupImage(name); err != nil {
t.Errorf("Error looking up %s: %s", name, err)
} else if img == nil {
t.Errorf("Expected 1 image, none found: %s", name)
} else if img.ID != testOfficialImageID {
t.Errorf("Expected ID '%s' found '%s'", testOfficialImageID, img.ID)
}
}
for _, name := range privateLookups {
if img, err := store.LookupImage(name); err != nil {
t.Errorf("Error looking up %s: %s", name, err)
} else if img == nil {
t.Errorf("Expected 1 image, none found: %s", name)
} else if img.ID != testPrivateImageID {
t.Errorf("Expected ID '%s' found '%s'", testPrivateImageID, img.ID)
}
}
for _, name := range invalidLookups {
if img, err := store.LookupImage(name); err == nil {
t.Errorf("Expected error, none found: %s", name)
} else if img != nil {
t.Errorf("Expected 0 image, 1 found: %s", name)
}
}
for _, name := range digestLookups {
if img, err := store.LookupImage(name); err != nil {
t.Errorf("Error looking up %s: %s", name, err)
} else if img == nil {
t.Errorf("Expected 1 image, none found: %s", name)
} else if img.ID != testPrivateImageID {
t.Errorf("Expected ID '%s' found '%s'", testPrivateImageID, img.ID)
}
}
}
func TestValidateDigest(t *testing.T) {
tests := []struct {
input string
expectError bool
}{
{"", true},
{"latest", true},
{"sha256:b", false},
{"tarsum+v1+sha256:bY852-_.+=", false},
{"#$%#$^:$%^#$%", true},
}
for i, test := range tests {
err := validateDigest(test.input)
gotError := err != nil
if e, a := test.expectError, gotError; e != a {
t.Errorf("%d: with input %s, expected error=%t, got %t: %s", i, test.input, test.expectError, gotError, err)
}
}
}