Introduce pkg/chunked/internal/path.RegularFilePathForValidatedDigest

... to centralize the composefs backing file layout design.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>

drivers/driver.go

@@ -231,8 +231,8 @@ const (
 	// DifferOutputFormatDir means the output is a directory and it will
 	// keep the original layout.
 	DifferOutputFormatDir = iota
-	// DifferOutputFormatFlat will store the files by their checksum, in the form
-	// checksum[0:2]/checksum[2:]
+	// DifferOutputFormatFlat will store the files by their checksum, per
+	// pkg/chunked/internal/composefs.RegularFilePathForValidatedDigest.
 	DifferOutputFormatFlat
 )
 

pkg/chunked/dump/dump.go

@@ -9,11 +9,11 @@ import (
 	"io"
 	"path/filepath"
 	"reflect"
-	"strings"
 	"time"
 
 	"github.com/containers/storage/pkg/chunked/internal/minimal"
 	storagePath "github.com/containers/storage/pkg/chunked/internal/path"
+	"github.com/opencontainers/go-digest"
 	"golang.org/x/sys/unix"
 )
 
@@ -165,11 +165,16 @@ func dumpNode(out io.Writer, added map[string]*minimal.FileMetadata, links map[s
 		} else {
 			payload = storagePath.CleanAbsPath(entry.Linkname)
 		}
-	} else {
-		if len(entry.Digest) > 10 {
-			d := strings.Replace(entry.Digest, "sha256:", "", 1)
-			payload = d[:2] + "/" + d[2:]
+	} else if entry.Digest != "" {
+		d, err := digest.Parse(entry.Digest)
+		if err != nil {
+			return fmt.Errorf("invalid digest %q for %q: %w", entry.Digest, entry.Name, err)
 		}
+		path, err := storagePath.RegularFilePathForValidatedDigest(d)
+		if err != nil {
+			return fmt.Errorf("determining physical file path for %q: %w", entry.Name, err)
+		}
+		payload = path
 	}
 
 	if _, err := fmt.Fprint(out, escapedOptional([]byte(payload), ESCAPE_LONE_DASH)); err != nil {

pkg/chunked/internal/path/path.go

@@ -1,7 +1,10 @@
 package path
 
 import (
+	"fmt"
 	"path/filepath"
+
+	"github.com/opencontainers/go-digest"
 )
 
 // CleanAbsPath removes any ".." and "." from the path
@@ -10,3 +13,15 @@ import (
 func CleanAbsPath(path string) string {
 	return filepath.Clean("/" + path)
 }
+
+// RegularFilePath returns the path used in the composefs backing store for a
+// regular file with the provided content digest.
+//
+// The caller MUST ensure d is a valid digest (in particular, that it contains no path separators or .. entries)
+func RegularFilePathForValidatedDigest(d digest.Digest) (string, error) {
+	if algo := d.Algorithm(); algo != digest.SHA256 {
+		return "", fmt.Errorf("unexpected digest algorithm %q", algo)
+	}
+	e := d.Encoded()
+	return e[0:2] + "/" + e[2:], nil
+}

pkg/chunked/internal/path/path_test.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/opencontainers/go-digest"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestCleanAbsPath(t *testing.T) {
@@ -46,3 +48,16 @@ func TestCleanAbsPath(t *testing.T) {
 		assert.Equal(t, test.expected, CleanAbsPath(test.path), fmt.Sprintf("path %q failed", test.path))
 	}
 }
+
+func TestRegularFilePathForValidatedDigest(t *testing.T) {
+	d, err := digest.Parse("sha256:0123456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef")
+	require.NoError(t, err)
+	res, err := RegularFilePathForValidatedDigest(d)
+	require.NoError(t, err)
+	assert.Equal(t, "01/23456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef", res)
+
+	d, err = digest.Parse("sha512:0123456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef0123456789abcdef1123456789abcdef2123456789abcdef3123456789abcdef")
+	require.NoError(t, err)
+	_, err = RegularFilePathForValidatedDigest(d)
+	assert.Error(t, err)
+}

pkg/chunked/storage_linux.go

@@ -1130,10 +1130,12 @@ func makeEntriesFlat(mergedEntries []fileMetadata) ([]fileMetadata, error) {
 		}
 		digest, err := digest.Parse(mergedEntries[i].Digest)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("invalid digest %q for %q: %w", mergedEntries[i].Digest, mergedEntries[i].Name, err)
+		}
+		path, err := path.RegularFilePathForValidatedDigest(digest)
+		if err != nil {
+			return nil, fmt.Errorf("determining physical file path for %q: %w", mergedEntries[i].Name, err)
 		}
-		d := digest.Encoded()
-		path := fmt.Sprintf("%s/%s", d[0:2], d[2:])
 
 		if _, known := knownFlatPaths[path]; known {
 			continue
@@ -1449,7 +1451,8 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 		}
 		createdDirs := make(map[string]struct{})
 		for _, e := range mergedEntries {
-			d := e.Name[0:2]
+			// This hard-codes an assumption that RegularFilePathForValidatedDigest creates paths with exactly one directory component.
+			d := filepath.Dir(e.Name)
 			if _, found := createdDirs[d]; !found {
 				if err := unix.Mkdirat(dirfd, d, 0o755); err != nil {
 					return output, &fs.PathError{Op: "mkdirat", Path: d, Err: err}