containers
/
storage
mirror of https://github.com/containers/storage.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chunked: add tests for the cache 

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>
This commit is contained in:
Giuseppe Scrivano 2022-01-13 12:09:53 +01:00
parent ab25eafc17
commit fd89b93ef3
No known key found for this signature in database
GPG Key ID: 67E38F7A8BA21772
2 changed files with 204 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
pkg/chunked/cache_linux.go
View File

@ -107,7 +107,16 @@ func (c *layersCache) load() error {
continue
}
metadata, err := c.readMetadataFromCache(r.ID)
bigData, err := c.store.LayerBigData(r.ID, cacheKey)
if err != nil {
if errors.Cause(err) == os.ErrNotExist {
continue
}
return err
}
defer bigData.Close()
metadata, err := readMetadataFromCache(bigData)
if err != nil {
logrus.Warningf("Error reading cache file for layer %q: %v", r.ID, err)
}
@ -117,7 +126,17 @@ func (c *layersCache) load() error {
continue
}
metadata, err = c.writeCache(r.ID)
manifestReader, err := c.store.LayerBigData(r.ID, bigDataKey)
if err != nil {
continue
}
defer manifestReader.Close()
manifest, err := ioutil.ReadAll(manifestReader)
if err != nil {
return fmt.Errorf("open manifest file for layer %q: %w", r.ID, err)
}
metadata, err = writeCache(manifest, r.ID, c.store)
if err == nil {
c.addLayer(r.ID, metadata)
}
@ -182,6 +201,11 @@ func generateTag(digest string, offset, len uint64) string {
return fmt.Sprintf("%s%.20d@%.20d", digest, offset, len)
}
type setBigData interface {
// SetLayerBigData stores a (possibly large) chunk of named data
SetLayerBigData(id, key string, data io.Reader) error
}
// writeCache write a cache for the layer ID.
// It generates a sorted list of digests with their offset to the path location and offset.
// The same cache is used to lookup files, chunks and candidates for deduplication with hard links.
@ -189,13 +213,13 @@ func generateTag(digest string, offset, len uint64) string {
// - digest(file.payload))
// - digest(digest(file.payload) + file.UID + file.GID + file.mode + file.xattrs)
// - digest(i) for each i in chunks(file payload)
func (c *layersCache) writeCache(id string) (*metadata, error) {
func writeCache(manifest []byte, id string, dest setBigData) (*metadata, error) {
var vdata bytes.Buffer
tagLen := 0
digestLen := 0
var tagsBuffer bytes.Buffer
toc, err := c.prepareMetadata(id)
toc, err := prepareMetadata(manifest)
if err != nil {
return nil, err
}
@ -317,7 +341,7 @@ func (c *layersCache) writeCache(id string) (*metadata, error) {
r := io.TeeReader(pipeReader, counter)
if err := c.store.SetLayerBigData(id, cacheKey, r); err != nil {
if err := dest.SetLayerBigData(id, cacheKey, r); err != nil {
return nil, err
}
@ -335,16 +359,7 @@ func (c *layersCache) writeCache(id string) (*metadata, error) {
}, nil
}
func (c *layersCache) readMetadataFromCache(id string) (*metadata, error) {
bigData, err := c.store.LayerBigData(id, cacheKey)
if err != nil {
if errors.Cause(err) == os.ErrNotExist {
return nil, nil
}
return nil, err
}
defer bigData.Close()
func readMetadataFromCache(bigData io.Reader) (*metadata, error) {
var version, tagLen, digestLen, tagsLen, vdataLen uint64
if err := binary.Read(bigData, binary.LittleEndian, &version); err != nil {
return nil, err
@ -371,7 +386,7 @@ func (c *layersCache) readMetadataFromCache(id string) (*metadata, error) {
}
vdata := make([]byte, vdataLen)
if _, err = bigData.Read(vdata); err != nil {
if _, err := bigData.Read(vdata); err != nil {
return nil, err
}
@ -383,17 +398,7 @@ func (c *layersCache) readMetadataFromCache(id string) (*metadata, error) {
}, nil
}
func (c *layersCache) prepareMetadata(id string) ([]*internal.FileMetadata, error) {
manifestReader, err := c.store.LayerBigData(id, bigDataKey)
if err != nil {
return nil, nil
}
defer manifestReader.Close()
manifest, err := ioutil.ReadAll(manifestReader)
if err != nil {
return nil, fmt.Errorf("open manifest file for layer %q: %w", id, err)
}
func prepareMetadata(manifest []byte) ([]*internal.FileMetadata, error) {
toc, err := unmarshalToc(manifest)
if err != nil {
// ignore errors here. They might be caused by a different manifest format.
@ -406,6 +411,7 @@ func (c *layersCache) prepareMetadata(id string) ([]*internal.FileMetadata, erro
d := toc.Entries[i].Digest
if d != "" {
r = append(r, &toc.Entries[i])
continue
}
// chunks do not use hard link dedup so keeping just one candidate is enough

171
pkg/chunked/cache_linux_test.go Normal file
View File

@ -0,0 +1,171 @@
package chunked
import (
"bytes"
"fmt"
"io"
"reflect"
"testing"
)
const jsonTOC = `
{
"version": 1,
"entries": [
{
"type": "symlink",
"name": "bin",
"linkName": "usr/bin",
"mode": 511,
"modtime": "1970-01-01T01:00:00+01:00",
"accesstime": "0001-01-01T00:00:00Z",
"changetime": "0001-01-01T00:00:00Z"
},
{
"type": "dir",
"name": "usr/bin",
"mode": 511,
"modtime": "2022-01-07T12:36:43+01:00",
"accesstime": "0001-01-01T00:00:00Z",
"changetime": "0001-01-01T00:00:00Z"
},
{
"type": "reg",
"name": "usr/bin/foo",
"mode": 511,
"size": 103867,
"modtime": "1970-01-01T01:00:00+01:00",
"accesstime": "0001-01-01T00:00:00Z",
"changetime": "0001-01-01T00:00:00Z",
"digest": "sha256:99fe908c699dc068438b23e28319cadff1f2153c3043bafb8e83a430bba0a2c6",
"offset": 94149,
"endOffset": 120135,
"chunkSize": 17615,
"chunkDigest": "sha256:2ce0d0f8eb2aa93d13007097763e4459c814c8d0e859e5a57465af924169b544"
},
{
"type": "chunk",
"name": "usr/bin/foo",
"offset": 99939,
"chunkSize": 86252,
"chunkOffset": 17615,
"chunkDigest": "sha256:2a9d3f1b6b37abc8bb35eb8fa98b893a2a2447bcb01184c3bafc8c6b40da099d"
}
}
`
func TestPrepareMetadata(t *testing.T) {
toc, err := prepareMetadata([]byte(jsonTOC))
if err != nil {
t.Errorf("got error from prepareMetadata: %w", err)
}
if len(toc) != 2 {
t.Error("prepareMetadata returns the wrong length")
}
}
type bigDataToBuffer struct {
buf *bytes.Buffer
id string
key string
called bool
}
func (b *bigDataToBuffer) SetLayerBigData(id, key string, data io.Reader) error {
b.id = id
b.key = key
if b.called {
return fmt.Errorf("SetLayerBigData already called once")
}
b.called = true
_, err := io.Copy(b.buf, data)
return err
}
func TestWriteCache(t *testing.T) {
toc, err := prepareMetadata([]byte(jsonTOC))
if err != nil {
t.Errorf("got error from prepareMetadata: %w", err)
}
dest := bigDataToBuffer{
buf: bytes.NewBuffer(nil),
}
cache, err := writeCache([]byte(jsonTOC), "foobar", &dest)
if err != nil {
t.Errorf("got error from writeCache: %w", err)
}
if digest, _, _ := findTag("foobar", cache); digest != "" {
t.Error("found invalid tag")
}
for _, r := range toc {
if r.Digest != "" {
// find the element in the cache by the digest checksum
digest, off, len := findTag(r.Digest, cache)
if digest == "" {
t.Error("file tag not found")
}
if digest != r.Digest {
t.Error("wrong file found")
}
expectedLocation := generateFileLocation(r.Name, 0)
location := cache.vdata[off : off+len]
if !bytes.Equal(location, expectedLocation) {
t.Errorf("wrong file found %q instead of %q", location, expectedLocation)
}
fingerprint, err := calculateHardLinkFingerprint(r)
if err != nil {
t.Errorf("got error from writeCache: %w", err)
}
// find the element in the cache by the hardlink fingerprint
digest, off, len = findTag(fingerprint, cache)
if digest == "" {
t.Error("file tag not found")
}
if digest != fingerprint {
t.Error("wrong file found")
}
expectedLocation = generateFileLocation(r.Name, 0)
location = cache.vdata[off : off+len]
if !bytes.Equal(location, expectedLocation) {
t.Errorf("wrong file found %q instead of %q", location, expectedLocation)
}
}
if r.ChunkDigest != "" {
// find the element in the cache by the chunk digest checksum
digest, off, len := findTag(r.ChunkDigest, cache)
if digest == "" {
t.Error("chunk tag not found")
}
if digest != r.ChunkDigest {
t.Error("wrong digest found")
}
expectedLocation := generateFileLocation(r.Name, uint64(r.ChunkOffset))
location := cache.vdata[off : off+len]
if !bytes.Equal(location, expectedLocation) {
t.Errorf("wrong file found %q instead of %q", location, expectedLocation)
}
}
}
}
func TestReadCache(t *testing.T) {
dest := bigDataToBuffer{
buf: bytes.NewBuffer(nil),
}
cache, err := writeCache([]byte(jsonTOC), "foobar", &dest)
if err != nil {
t.Errorf("got error from writeCache: %w", err)
}
cacheRead, err := readMetadataFromCache(dest.buf)
if err != nil {
t.Errorf("got error from readMetadataFromCache: %w", err)
}
if !reflect.DeepEqual(cache, cacheRead) {
t.Errorf("read a different struct than what was written")
}
}