Merge pull request #8393 from dmcgowan/provenance_pull_enhance

Make V2 code more defensive against malformed content

graph/pull.go

@@ -39,6 +39,9 @@ func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*re
 	if err := json.Unmarshal(payload, &manifest); err != nil {
 		return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err)
 	}
+	if manifest.SchemaVersion != 1 {
+		return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion)
+	}
 
 	var verified bool
 	for _, key := range keys {
@@ -134,6 +137,19 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
 		mirrors = s.mirrors
 	}
 
+	if isOfficial || endpoint.Version == registry.APIVersion2 {
+		j := job.Eng.Job("trust_update_base")
+		if err = j.Run(); err != nil {
+			return job.Errorf("error updating trust base graph: %s", err)
+		}
+
+		if err := s.pullV2Repository(job.Eng, r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err == nil {
+			return engine.StatusOK
+		} else if err != registry.ErrDoesNotExist {
+			log.Errorf("Error from V2 registry: %s", err)
+		}
+	}
+
 	if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil {
 		return job.Error(err)
 	}
@@ -413,6 +429,7 @@ type downloadInfo struct {
 }
 
 func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error {
+	var layersDownloaded bool
 	if tag == "" {
 		log.Debugf("Pulling tag list from V2 registry for %s", remoteName)
 		tags, err := r.GetV2RemoteTags(remoteName, nil)
@@ -420,51 +437,65 @@ func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out
 			return err
 		}
 		for _, t := range tags {
-			if err := s.pullV2Tag(eng, r, out, localName, remoteName, t, sf, parallel); err != nil {
+			if downloaded, err := s.pullV2Tag(eng, r, out, localName, remoteName, t, sf, parallel); err != nil {
 				return err
+			} else if downloaded {
+				layersDownloaded = true
 			}
 		}
 	} else {
-		if err := s.pullV2Tag(eng, r, out, localName, remoteName, tag, sf, parallel); err != nil {
+		if downloaded, err := s.pullV2Tag(eng, r, out, localName, remoteName, tag, sf, parallel); err != nil {
 			return err
+		} else if downloaded {
+			layersDownloaded = true
 		}
 	}
 
+	requestedTag := localName
+	if len(tag) > 0 {
+		requestedTag = localName + ":" + tag
+	}
+	WriteStatus(requestedTag, out, sf, layersDownloaded)
 	return nil
 }
 
-func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error {
+func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) (bool, error) {
 	log.Debugf("Pulling tag from V2 registry: %q", tag)
 	manifestBytes, err := r.GetV2ImageManifest(remoteName, tag, nil)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	manifest, verified, err := s.verifyManifest(eng, manifestBytes)
 	if err != nil {
-		return fmt.Errorf("error verifying manifest: %s", err)
+		return false, fmt.Errorf("error verifying manifest: %s", err)
 	}
 
-	if len(manifest.BlobSums) != len(manifest.History) {
+	if len(manifest.FSLayers) != len(manifest.History) {
-		return fmt.Errorf("length of history not equal to number of layers")
+		return false, fmt.Errorf("length of history not equal to number of layers")
 	}
 
 	if verified {
-		out.Write(sf.FormatStatus("", "The image you are pulling has been digitally signed by Docker, Inc."))
+		out.Write(sf.FormatStatus(localName+":"+tag, "The image you are pulling has been verified"))
+	} else {
+		out.Write(sf.FormatStatus(tag, "Pulling from %s", localName))
 	}
-	out.Write(sf.FormatStatus(tag, "Pulling from %s", localName))
 
-	downloads := make([]downloadInfo, len(manifest.BlobSums))
+	if len(manifest.FSLayers) == 0 {
+		return false, fmt.Errorf("no blobSums in manifest")
+	}
 
-	for i := len(manifest.BlobSums) - 1; i >= 0; i-- {
+	downloads := make([]downloadInfo, len(manifest.FSLayers))
+
+	for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
 		var (
-			sumStr  = manifest.BlobSums[i]
+			sumStr  = manifest.FSLayers[i].BlobSum
-			imgJSON = []byte(manifest.History[i])
+			imgJSON = []byte(manifest.History[i].V1Compatibility)
 		)
 
 		img, err := image.NewImgJSON(imgJSON)
 		if err != nil {
-			return fmt.Errorf("failed to parse json: %s", err)
+			return false, fmt.Errorf("failed to parse json: %s", err)
 		}
 		downloads[i].img = img
 
@@ -476,13 +507,13 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
 
 		chunks := strings.SplitN(sumStr, ":", 2)
 		if len(chunks) < 2 {
-			return fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks)
+			return false, fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks)
 		}
 		sumType, checksum := chunks[0], chunks[1]
 		out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling fs layer", nil))
 
 		downloadFunc := func(di *downloadInfo) error {
-			log.Infof("pulling blob %q to V1 img %s", sumStr, img.ID)
+			log.Debugf("pulling blob %q to V1 img %s", sumStr, img.ID)
 
 			if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil {
 				if c != nil {
@@ -493,6 +524,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
 					log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
 				}
 			} else {
+				defer s.poolRemove("pull", "img:"+img.ID)
 				tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob")
 				if err != nil {
 					return err
@@ -513,7 +545,6 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
 				di.downloaded = true
 			}
 			di.imgJSON = imgJSON
-			defer s.poolRemove("pull", "img:"+img.ID)
 
 			return nil
 		}
@@ -526,17 +557,18 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
 		} else {
 			err := downloadFunc(&downloads[i])
 			if err != nil {
-				return err
+				return false, err
 			}
 		}
 	}
 
+	var layersDownloaded bool
 	for i := len(downloads) - 1; i >= 0; i-- {
 		d := &downloads[i]
 		if d.err != nil {
 			err := <-d.err
 			if err != nil {
-				return err
+				return false, err
 			}
 		}
 		if d.downloaded {
@@ -548,13 +580,13 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
 				err = s.graph.Register(d.img, d.imgJSON,
 					utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting"))
 				if err != nil {
-					return err
+					return false, err
 				}
 
 				// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
 			}
 			out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Pull complete", nil))
-
+			layersDownloaded = true
 		} else {
 			out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Already exists", nil))
 		}
@@ -562,8 +594,8 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
 	}
 
 	if err = s.Set(localName, tag, downloads[0].img.ID, true); err != nil {
-		return err
+		return false, err
 	}
 
-	return nil
+	return layersDownloaded, nil
 }

registry/auth.go

@@ -14,13 +14,16 @@ import (
 	"github.com/docker/docker/utils"
 )
 
-// Where we store the config file
+const (
-const CONFIGFILE = ".dockercfg"
+	// Where we store the config file
+	CONFIGFILE = ".dockercfg"
 
-// Only used for user auth + account creation
+	// Only used for user auth + account creation
-const INDEXSERVER = "https://index.docker.io/v1/"
+	INDEXSERVER    = "https://index.docker.io/v1/"
+	REGISTRYSERVER = "https://registry-1.docker.io/v1/"
 
-//const INDEXSERVER = "https://registry-stage.hub.docker.com/v1/"
+	// INDEXSERVER = "https://registry-stage.hub.docker.com/v1/"
+)
 
 var (
 	ErrConfigFileMissing = errors.New("The Auth config file is missing")

registry/session_v2.go

@@ -28,13 +28,13 @@ func newV2RegistryRouter() *mux.Router {
 	v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags")
 
 	// Download a blob
-	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob")
+	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob")
 
 	// Upload a blob
-	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name("uploadBlob")
+	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob")
 
 	// Mounting a blob in an image
-	v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob")
+	v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob")
 
 	return router
 }
@@ -57,10 +57,14 @@ func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL,
 	if err != nil {
 		return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err)
 	}
+	u, err := url.Parse(REGISTRYSERVER)
+	if err != nil {
+		return nil, fmt.Errorf("invalid registry url: %s", err)
+	}
 
 	return &url.URL{
-		Scheme: e.URL.Scheme,
+		Scheme: u.Scheme,
-		Host:   e.URL.Host,
+		Host:   u.Host,
 		Path:   routePath.Path,
 	}, nil
 }

registry/types.go

@@ -32,13 +32,21 @@ type RegistryInfo struct {
 	Standalone bool   `json:"standalone"`
 }
 
+type FSLayer struct {
+	BlobSum string `json:"blobSum"`
+}
+
+type ManifestHistory struct {
+	V1Compatibility string `json:"v1Compatibility"`
+}
+
 type ManifestData struct {
-	Name          string   `json:"name"`
+	Name          string             `json:"name"`
-	Tag           string   `json:"tag"`
+	Tag           string             `json:"tag"`
-	Architecture  string   `json:"architecture"`
+	Architecture  string             `json:"architecture"`
-	BlobSums      []string `json:"blobSums"`
+	FSLayers      []*FSLayer         `json:"fsLayers"`
-	History       []string `json:"history"`
+	History       []*ManifestHistory `json:"history"`
-	SchemaVersion int      `json:"schemaVersion"`
+	SchemaVersion int                `json:"schemaVersion"`
 }
 
 type APIVersion int