288 lines
8.3 KiB
Go
288 lines
8.3 KiB
Go
package chunked
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
graphdriver "github.com/containers/storage/drivers"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
const jsonTOC = `
|
|
{
|
|
"version": 1,
|
|
"entries": [
|
|
{
|
|
"type": "symlink",
|
|
"name": "bin",
|
|
"linkName": "usr/bin",
|
|
"mode": 511,
|
|
"modtime": "1970-01-01T01:00:00+01:00",
|
|
"accesstime": "0001-01-01T00:00:00Z",
|
|
"changetime": "0001-01-01T00:00:00Z"
|
|
},
|
|
{
|
|
"type": "dir",
|
|
"name": "usr/bin",
|
|
"mode": 511,
|
|
"modtime": "2022-01-07T12:36:43+01:00",
|
|
"accesstime": "0001-01-01T00:00:00Z",
|
|
"changetime": "0001-01-01T00:00:00Z"
|
|
},
|
|
{
|
|
"type": "reg",
|
|
"name": "usr/bin/foo",
|
|
"mode": 511,
|
|
"size": 103867,
|
|
"modtime": "1970-01-01T01:00:00+01:00",
|
|
"accesstime": "0001-01-01T00:00:00Z",
|
|
"changetime": "0001-01-01T00:00:00Z",
|
|
"digest": "sha256:99fe908c699dc068438b23e28319cadff1f2153c3043bafb8e83a430bba0a2c6",
|
|
"offset": 94149,
|
|
"endOffset": 120135,
|
|
"chunkSize": 17615,
|
|
"chunkDigest": "sha256:2ce0d0f8eb2aa93d13007097763e4459c814c8d0e859e5a57465af924169b544"
|
|
},
|
|
{
|
|
"type": "chunk",
|
|
"name": "usr/bin/foo",
|
|
"offset": 99939,
|
|
"chunkSize": 86252,
|
|
"chunkOffset": 17615,
|
|
"chunkDigest": "sha256:2a9d3f1b6b37abc8bb35eb8fa98b893a2a2447bcb01184c3bafc8c6b40da099d"
|
|
},
|
|
{
|
|
"type": "reg",
|
|
"name": "usr/lib/systemd/system/system-systemd\\x2dcryptsetup.slice",
|
|
"mode": 420,
|
|
"size": 468,
|
|
"modtime": "2024-03-03T18:04:57+01:00",
|
|
"accesstime": "0001-01-01T00:00:00Z",
|
|
"changetime": "0001-01-01T00:00:00Z",
|
|
"digest": "sha256:68dc6e85631e077f2bc751352459823844911b93b7ba2afd95d96c893222bb50",
|
|
"offset": 148185424,
|
|
"endOffset": 148185753
|
|
},
|
|
{
|
|
"type": "reg",
|
|
"name": "usr/lib/systemd/system/system-systemd\\x2dcryptsetup-hardlink.slice",
|
|
"linkName": "usr/lib/systemd/system/system-systemd\\x2dcryptsetup.slice"
|
|
}
|
|
]
|
|
}
|
|
`
|
|
|
|
func TestPrepareMetadata(t *testing.T) {
|
|
toc, err := prepareCacheFile([]byte(jsonTOC), graphdriver.DifferOutputFormatDir)
|
|
if err != nil {
|
|
t.Errorf("got error from prepareCacheFile: %v", err)
|
|
}
|
|
if len(toc) != 4 {
|
|
t.Error("prepareCacheFile returns the wrong length")
|
|
}
|
|
}
|
|
|
|
func TestPrepareMetadataFlat(t *testing.T) {
|
|
toc, err := prepareCacheFile([]byte(jsonTOC), graphdriver.DifferOutputFormatFlat)
|
|
if err != nil {
|
|
t.Errorf("got error from prepareCacheFile: %v", err)
|
|
}
|
|
for _, e := range toc {
|
|
if len(strings.Split(e.Name, "/")) != 2 {
|
|
t.Error("prepareCacheFile returns the wrong number of path elements for flat directories")
|
|
}
|
|
if len(filepath.Dir(e.Name)) != 2 {
|
|
t.Error("prepareCacheFile returns the wrong path for flat directories")
|
|
}
|
|
}
|
|
}
|
|
|
|
type bigDataToBuffer struct {
|
|
buf *bytes.Buffer
|
|
id string
|
|
key string
|
|
called bool
|
|
}
|
|
|
|
func (b *bigDataToBuffer) SetLayerBigData(id, key string, data io.Reader) error {
|
|
b.id = id
|
|
b.key = key
|
|
if b.called {
|
|
return fmt.Errorf("SetLayerBigData already called once")
|
|
}
|
|
b.called = true
|
|
_, err := io.Copy(b.buf, data)
|
|
return err
|
|
}
|
|
|
|
func findTag(digest string, cacheFile *cacheFile) (string, uint64, uint64) {
|
|
binaryDigest, err := makeBinaryDigest(digest)
|
|
if err != nil {
|
|
return "", 0, 0
|
|
}
|
|
if len(binaryDigest) != cacheFile.digestLen {
|
|
return "", 0, 0
|
|
}
|
|
found, off, len := findBinaryTag(binaryDigest, cacheFile)
|
|
if found {
|
|
return digest, off, len
|
|
}
|
|
return "", 0, 0
|
|
}
|
|
|
|
func TestWriteCache(t *testing.T) {
|
|
toc, err := prepareCacheFile([]byte(jsonTOC), graphdriver.DifferOutputFormatDir)
|
|
if err != nil {
|
|
t.Errorf("got error from prepareCacheFile: %v", err)
|
|
}
|
|
|
|
dest := bigDataToBuffer{
|
|
buf: bytes.NewBuffer(nil),
|
|
}
|
|
cache, err := writeCache([]byte(jsonTOC), graphdriver.DifferOutputFormatDir, "foobar", &dest)
|
|
if err != nil {
|
|
t.Errorf("got error from writeCache: %v", err)
|
|
}
|
|
if digest, _, _ := findTag("sha256:99fe908c699dc068438b23e28319cadff1f2153c3043bafb8e83a430bba0a2c2", cache); digest != "" {
|
|
t.Error("a present tag was not found")
|
|
}
|
|
|
|
for _, r := range toc {
|
|
if r.Digest != "" {
|
|
// find the element in the cache by the digest checksum
|
|
digest, off, lenTag := findTag(r.Digest, cache)
|
|
if digest == "" {
|
|
t.Error("file tag not found")
|
|
}
|
|
if digest != r.Digest {
|
|
t.Error("wrong file found")
|
|
}
|
|
location := cache.vdata[off : off+lenTag]
|
|
_, offFile, fileSize, err := parseFileLocation(location)
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, fileSize, uint64(r.Size))
|
|
assert.Equal(t, offFile, uint64(0))
|
|
|
|
fingerprint, err := calculateHardLinkFingerprint(r)
|
|
if err != nil {
|
|
t.Errorf("got error from writeCache: %v", err)
|
|
}
|
|
|
|
// find the element in the cache by the hardlink fingerprint
|
|
digest, off, lenTag = findTag(fingerprint, cache)
|
|
if digest == "" {
|
|
t.Error("file tag not found")
|
|
}
|
|
if digest != fingerprint {
|
|
t.Error("wrong file found")
|
|
}
|
|
location = cache.vdata[off : off+lenTag]
|
|
_, offFile, fileSize, err = parseFileLocation(location)
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, fileSize, uint64(r.Size))
|
|
assert.Equal(t, offFile, uint64(0))
|
|
}
|
|
if r.ChunkDigest != "" {
|
|
// find the element in the cache by the chunk digest checksum
|
|
digest, off, len := findTag(r.ChunkDigest, cache)
|
|
if digest == "" {
|
|
t.Error("chunk tag not found")
|
|
}
|
|
if digest != r.ChunkDigest {
|
|
t.Error("wrong digest found")
|
|
}
|
|
expectedLocation := generateFileLocation(0, uint64(r.ChunkOffset), uint64(r.ChunkSize))
|
|
location := cache.vdata[off : off+len]
|
|
if !bytes.Equal(location, expectedLocation) {
|
|
t.Errorf("wrong file found %q instead of %q", location, expectedLocation)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReadCache(t *testing.T) {
|
|
dest := bigDataToBuffer{
|
|
buf: bytes.NewBuffer(nil),
|
|
}
|
|
cache, err := writeCache([]byte(jsonTOC), graphdriver.DifferOutputFormatDir, "foobar", &dest)
|
|
if err != nil {
|
|
t.Errorf("got error from writeCache: %v", err)
|
|
}
|
|
|
|
cacheRead, err := readCacheFileFromMemory(dest.buf.Bytes())
|
|
if err != nil {
|
|
t.Errorf("got error from readMetadataFromCache: %v", err)
|
|
}
|
|
if !reflect.DeepEqual(cache, cacheRead) {
|
|
t.Errorf("read a different struct than what was written")
|
|
}
|
|
}
|
|
|
|
func FuzzReadCache(f *testing.F) {
|
|
dest := &bigDataToBuffer{
|
|
buf: bytes.NewBuffer(nil),
|
|
}
|
|
_, err := writeCache([]byte(jsonTOC), graphdriver.DifferOutputFormatDir, "foobar", dest)
|
|
if err != nil {
|
|
f.Errorf("got error from writeCache: %v", err)
|
|
}
|
|
|
|
f.Add(dest.buf.Bytes())
|
|
dest = nil
|
|
|
|
f.Fuzz(func(t *testing.T, orig []byte) {
|
|
cacheRead, err := readCacheFileFromMemory(orig)
|
|
if err != nil || cacheRead == nil {
|
|
return
|
|
}
|
|
findTag("", cacheRead)
|
|
findTag("foo", cacheRead)
|
|
findTag("foo:bar", cacheRead)
|
|
findTag("sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", cacheRead)
|
|
})
|
|
}
|
|
|
|
func TestUnmarshalToc(t *testing.T) {
|
|
toc, err := unmarshalToc([]byte(jsonTOC))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 6, len(toc.Entries))
|
|
|
|
_, err = unmarshalToc([]byte(jsonTOC + " \n\n\n\n "))
|
|
assert.NoError(t, err)
|
|
_, err = unmarshalToc([]byte(jsonTOC + "aaaa"))
|
|
assert.Error(t, err)
|
|
_, err = unmarshalToc([]byte(jsonTOC + ","))
|
|
assert.Error(t, err)
|
|
_, err = unmarshalToc([]byte(jsonTOC + "{}"))
|
|
assert.Error(t, err)
|
|
_, err = unmarshalToc([]byte(jsonTOC + "[]"))
|
|
assert.Error(t, err)
|
|
_, err = unmarshalToc([]byte(jsonTOC + "\"aaaa\""))
|
|
assert.Error(t, err)
|
|
_, err = unmarshalToc([]byte(jsonTOC + "123"))
|
|
assert.Error(t, err)
|
|
assert.Equal(t, toc.Entries[4].Name, "usr/lib/systemd/system/system-systemd\\x2dcryptsetup.slice", "invalid name escaped")
|
|
assert.Equal(t, toc.Entries[5].Name, "usr/lib/systemd/system/system-systemd\\x2dcryptsetup-hardlink.slice", "invalid name escaped")
|
|
assert.Equal(t, toc.Entries[5].Linkname, "usr/lib/systemd/system/system-systemd\\x2dcryptsetup.slice", "invalid link name escaped")
|
|
}
|
|
|
|
func TestMakeBinaryDigest(t *testing.T) {
|
|
binDigest, err := makeBinaryDigest("sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03")
|
|
assert.NoError(t, err)
|
|
expected := []byte{0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x3a, 0x58, 0x91, 0xb5, 0xb5, 0x22, 0xd5, 0xdf, 0x8, 0x6d, 0xf, 0xf0, 0xb1, 0x10, 0xfb, 0xd9, 0xd2, 0x1b, 0xb4, 0xfc, 0x71, 0x63, 0xaf, 0x34, 0xd0, 0x82, 0x86, 0xa2, 0xe8, 0x46, 0xf6, 0xbe, 0x3}
|
|
assert.Equal(t, expected, binDigest)
|
|
|
|
_, err = makeBinaryDigest("sha256:foo")
|
|
assert.Error(t, err)
|
|
|
|
_, err = makeBinaryDigest("noAlgorithm")
|
|
assert.Error(t, err)
|
|
}
|