Rework how we use storage for storing images

This patch overhauls how we use containers/storage to store images,
dropping most of our own metadata in favor of facilities that are
now provided by default by the storage library.  Additionally:
* storageImageDestination now caches blobs in a temporary directory
  until Commit() is called
* storageImageDestination generates a barebones manifest if one isn't
  supplied before Commit() is called
* storageImageDestination uses new APIs in containers/storage to
  look for a local layer with the same contents of a blob, making it
  better at noticing when a PutBlob() isn't necessary
* storageImageDestination sets the creation date for the image if it
  can be determined during Commit()
* storageImageDestination defaults to using the hex part of the digest
  of the image's configuration blob as an image's ID, making it better
  at catching re-pulls of the same image
* storageImageDestination no longer discards names which have been set
  for an image when reusing an ID
* storageImage now counts sizes of uncompressed data when determining
  image size
* storageImage now counts the size of the configuration blob when
  computing an image's size
* storageImage returns an updated image with the manifest listing
  uncompressed layer blobs
* storageImageSource also returns such an updated manifest
* storageImageSource now always returns uncompressed layers

Test changes:
* storage tests now always write an image manifest
* the test for determining an image's size now actually writes the
  configuration blob that it later tries to read

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

storage/storage_reference.go

@@ -162,9 +162,9 @@ func (s storageReference) DeleteImage(ctx *types.SystemContext) error {
 }
 
 func (s storageReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
-	return newImageSource(s)
+	return newImageSource(ctx, s)
 }
 
 func (s storageReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) {
-	return newImageDestination(s)
+	return newImageDestination(ctx, s)
 }

storage/storage_test.go

@@ -444,14 +444,11 @@ func TestWriteRead(t *testing.T) {
 				t.Fatalf("NewImageSource(%q) changed the reference to %q", ref.StringWithinTransport(), src.Reference().StringWithinTransport())
 			}
 		}
-		retrievedManifest, manifestType, err := src.GetManifest(nil)
+		_, manifestType, err := src.GetManifest(nil)
 		if err != nil {
 			t.Fatalf("GetManifest(%q) returned error %v", ref.StringWithinTransport(), err)
 		}
 		t.Logf("this manifest's type appears to be %q", manifestType)
-		if string(retrievedManifest) != manifest {
-			t.Fatalf("NewImageSource(%q) changed the manifest: %q was %q", ref.StringWithinTransport(), string(retrievedManifest), manifest)
-		}
 		sum = ddigest.SHA256.FromBytes([]byte(manifest))
 		_, _, err = src.GetManifest(&sum)
 		if err == nil {
@@ -511,6 +508,75 @@ func TestWriteRead(t *testing.T) {
 	}
 }
 
+func TestCommitWithoutManifest(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip("TestCommitWithoutManifest requires root privileges")
+	}
+
+	newStore(t)
+
+	ref, err := Transport.ParseReference("test")
+	if err != nil {
+		t.Fatalf("ParseReference(%q) returned error %v", "test", err)
+	}
+	if ref == nil {
+		t.Fatalf("ParseReference returned nil reference")
+	}
+
+	dest, err := ref.NewImageDestination(systemContext())
+	if err != nil {
+		t.Fatalf("NewImageDestination(%q) returned error %v", ref.StringWithinTransport(), err)
+	}
+	if dest == nil {
+		t.Fatalf("NewImageDestination(%q) returned no destination", ref.StringWithinTransport())
+	}
+	blobs := [][]byte{}
+	infos := []types.BlobInfo{}
+	blobDigest, _, blobSize, blob := makeLayer(t, archive.Uncompressed)
+	blobs = append(blobs, blob)
+	infos = append(infos, types.BlobInfo{Size: blobSize, Digest: blobDigest})
+	blobDigest, _, blobSize, blob = makeLayer(t, archive.Uncompressed)
+	blobs = append(blobs, blob)
+	infos = append(infos, types.BlobInfo{Size: blobSize, Digest: blobDigest})
+	blobs = append(blobs, blobs[0])
+	infos = append(infos, infos[0])
+	blobDigest, _, blobSize, blob = makeLayer(t, archive.Uncompressed)
+	blobs = append(blobs, blob)
+	infos = append(infos, types.BlobInfo{Size: blobSize, Digest: blobDigest})
+	blobs = append(blobs, blob)
+	infos = append(infos, types.BlobInfo{Size: blobSize, Digest: blobDigest})
+	blobs = append(blobs, blobs[0])
+	infos = append(infos, infos[0])
+	for i := range blobs {
+		if _, err := dest.PutBlob(bytes.NewBuffer(blobs[i]), infos[i]); err != nil {
+			t.Fatalf("Error saving randomly-generated layer %d to destination: %v", i+1, err)
+		}
+	}
+	if err := dest.Commit(); err != nil {
+		t.Fatalf("Error committing changes to destination: %v", err)
+	}
+	dest.Close()
+
+	img, err := ref.NewImage(systemContext())
+	if err != nil {
+		t.Fatalf("NewImage(%q) returned error %v", ref.StringWithinTransport(), err)
+	}
+	if img == nil {
+		t.Fatalf("NewImage(%q) returned no destination", ref.StringWithinTransport())
+	}
+	infos = img.LayerInfos()
+	if len(infos) != len(blobs) {
+		t.Fatalf("Image(%q) had the wrong number of layers (expected 5, have %d)", ref.StringWithinTransport(), len(infos))
+	}
+	for i := range infos {
+		sum := ddigest.FromBytes(blobs[i])
+		if infos[i].Digest != sum {
+			t.Fatalf("Image(%q) layer %d was wrong (expected %q, have %q)", ref.StringWithinTransport(), i+1, sum, infos[i].Digest)
+		}
+	}
+	img.Close()
+}
+
 func TestDuplicateName(t *testing.T) {
 	if os.Geteuid() != 0 {
 		t.Skip("TestDuplicateName requires root privileges")
@@ -540,6 +606,22 @@ func TestDuplicateName(t *testing.T) {
 	}); err != nil {
 		t.Fatalf("Error saving randomly-generated layer to destination, first pass: %v", err)
 	}
+	manifest := fmt.Sprintf(`
+	        {
+		    "schemaVersion": 2,
+		    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+		    "layers": [
+			{
+			    "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+			    "digest": "%s",
+			    "size": %d
+			}
+		    ]
+		}
+	`, digest, size)
+	if err := dest.PutManifest([]byte(manifest)); err != nil {
+		t.Fatalf("Error storing manifest to destination: %v", err)
+	}
 	if err := dest.Commit(); err != nil {
 		t.Fatalf("Error committing changes to destination, first pass: %v", err)
 	}
@@ -559,6 +641,22 @@ func TestDuplicateName(t *testing.T) {
 	}); err != nil {
 		t.Fatalf("Error saving randomly-generated layer to destination, second pass: %v", err)
 	}
+	manifest = fmt.Sprintf(`
+	        {
+		    "schemaVersion": 2,
+		    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+		    "layers": [
+			{
+			    "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+			    "digest": "%s",
+			    "size": %d
+			}
+		    ]
+		}
+	`, digest, size)
+	if err := dest.PutManifest([]byte(manifest)); err != nil {
+		t.Fatalf("Error storing manifest to destination: %v", err)
+	}
 	if err := dest.Commit(); err != nil {
 		t.Fatalf("Error committing changes to destination, second pass: %v", err)
 	}
@@ -594,6 +692,22 @@ func TestDuplicateID(t *testing.T) {
 	}); err != nil {
 		t.Fatalf("Error saving randomly-generated layer to destination, first pass: %v", err)
 	}
+	manifest := fmt.Sprintf(`
+	        {
+		    "schemaVersion": 2,
+		    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+		    "layers": [
+			{
+			    "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+			    "digest": "%s",
+			    "size": %d
+			}
+		    ]
+		}
+	`, digest, size)
+	if err := dest.PutManifest([]byte(manifest)); err != nil {
+		t.Fatalf("Error storing manifest to destination: %v", err)
+	}
 	if err := dest.Commit(); err != nil {
 		t.Fatalf("Error committing changes to destination, first pass: %v", err)
 	}
@@ -613,6 +727,22 @@ func TestDuplicateID(t *testing.T) {
 	}); err != nil {
 		t.Fatalf("Error saving randomly-generated layer to destination, second pass: %v", err)
 	}
+	manifest = fmt.Sprintf(`
+	        {
+		    "schemaVersion": 2,
+		    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+		    "layers": [
+			{
+			    "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+			    "digest": "%s",
+			    "size": %d
+			}
+		    ]
+		}
+	`, digest, size)
+	if err := dest.PutManifest([]byte(manifest)); err != nil {
+		t.Fatalf("Error storing manifest to destination: %v", err)
+	}
 	if err := dest.Commit(); errors.Cause(err) != storage.ErrDuplicateID {
 		if err != nil {
 			t.Fatalf("Wrong error committing changes to destination, second pass: %v", err)
@@ -651,6 +781,22 @@ func TestDuplicateNameID(t *testing.T) {
 	}); err != nil {
 		t.Fatalf("Error saving randomly-generated layer to destination, first pass: %v", err)
 	}
+	manifest := fmt.Sprintf(`
+	        {
+		    "schemaVersion": 2,
+		    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+		    "layers": [
+			{
+			    "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+			    "digest": "%s",
+			    "size": %d
+			}
+		    ]
+		}
+	`, digest, size)
+	if err := dest.PutManifest([]byte(manifest)); err != nil {
+		t.Fatalf("Error storing manifest to destination: %v", err)
+	}
 	if err := dest.Commit(); err != nil {
 		t.Fatalf("Error committing changes to destination, first pass: %v", err)
 	}
@@ -670,6 +816,22 @@ func TestDuplicateNameID(t *testing.T) {
 	}); err != nil {
 		t.Fatalf("Error saving randomly-generated layer to destination, second pass: %v", err)
 	}
+	manifest = fmt.Sprintf(`
+	        {
+		    "schemaVersion": 2,
+		    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+		    "layers": [
+			{
+			    "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+			    "digest": "%s",
+			    "size": %d
+			}
+		    ]
+		}
+	`, digest, size)
+	if err := dest.PutManifest([]byte(manifest)); err != nil {
+		t.Fatalf("Error storing manifest to destination: %v", err)
+	}
 	if err := dest.Commit(); errors.Cause(err) != storage.ErrDuplicateID {
 		if err != nil {
 			t.Fatalf("Wrong error committing changes to destination, second pass: %v", err)
@@ -747,14 +909,17 @@ func TestSize(t *testing.T) {
 	if dest == nil {
 		t.Fatalf("NewImageDestination(%q) returned no destination", ref.StringWithinTransport())
 	}
-	digest1, _, size1, blob := makeLayer(t, archive.Gzip)
+	if _, err := dest.PutBlob(bytes.NewBufferString(config), configInfo); err != nil {
+		t.Fatalf("Error saving config to destination: %v", err)
+	}
+	digest1, usize1, size1, blob := makeLayer(t, archive.Gzip)
 	if _, err := dest.PutBlob(bytes.NewBuffer(blob), types.BlobInfo{
 		Size:   size1,
 		Digest: digest1,
 	}); err != nil {
 		t.Fatalf("Error saving randomly-generated layer 1 to destination: %v", err)
 	}
-	digest2, _, size2, blob := makeLayer(t, archive.Gzip)
+	digest2, usize2, size2, blob := makeLayer(t, archive.Gzip)
 	if _, err := dest.PutBlob(bytes.NewBuffer(blob), types.BlobInfo{
 		Size:   size2,
 		Digest: digest2,
@@ -800,8 +965,8 @@ func TestSize(t *testing.T) {
 	if usize == -1 || err != nil {
 		t.Fatalf("Error calculating image size: %v", err)
 	}
-	if int(usize) != layerSize*2+len(manifest) {
+	if int(usize) != len(config)+int(usize1)+int(usize2)+len(manifest) {
-		t.Fatalf("Unexpected image size: %d != %d + %d + %d", usize, layerSize, layerSize, len(manifest))
+		t.Fatalf("Unexpected image size: %d != %d + %d + %d + %d", usize, len(config), usize1, usize2, len(manifest))
 	}
 	img.Close()
 }
@@ -916,7 +1081,7 @@ func TestDuplicateBlob(t *testing.T) {
 	}
 	layers := []string{}
 	for _, layerInfo := range img.LayerInfos() {
-		rc, _, layerID, err := source.getBlobAndLayerID(layerInfo)
+		rc, _, layerID, err := source.image.reader.getBlobAndLayerID(layerInfo)
 		if err != nil {
 			t.Fatalf("getBlobAndLayerID(%q) returned error %v", layerInfo.Digest, err)
 		}