containers
/
automation-tests
mirror of https://github.com/containers/automation-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chunked: fix generating footer 

commit 8ef163a990 introduces an error
where the generated footer is not correct.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>
This commit is contained in:
Giuseppe Scrivano 2023-08-30 12:48:55 +02:00
parent 2c0ba1a132
commit d39258c12e
2 changed files with 57 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
storage/pkg/chunked/internal/compression.go
View File

@ -100,7 +100,7 @@ const (
// FooterSizeSupported is the footer size supported by this implementation.
// Newer versions of the image format might increase this value, so reject
// any version that is not supported.
FooterSizeSupported = 56
FooterSizeSupported = 64
)
var (
@ -109,7 +109,7 @@ var (
// https://tools.ietf.org/html/rfc8478#section-3.1.2
skippableFrameMagic = []byte{0x50, 0x2a, 0x4d, 0x18}
ZstdChunkedFrameMagic = []byte{0x47, 0x6e, 0x55, 0x6c, 0x49, 0x6e, 0x55, 0x78}
ZstdChunkedFrameMagic = []byte{0x47, 0x4e, 0x55, 0x6c, 0x49, 0x6e, 0x55, 0x78}
)
func appendZstdSkippableFrame(dest io.Writer, data []byte) error {
@ -184,13 +184,19 @@ func WriteZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, off
return err
}
// Store the offset to the manifest and its size in LE order
manifestDataLE := make([]byte, FooterSizeSupported)
binary.LittleEndian.PutUint64(manifestDataLE, manifestOffset)
binary.LittleEndian.PutUint64(manifestDataLE[8*1:], uint64(len(compressedManifest)))
binary.LittleEndian.PutUint64(manifestDataLE[8*2:], uint64(len(manifest)))
binary.LittleEndian.PutUint64(manifestDataLE[8*3:], uint64(ManifestTypeCRFS))
copy(manifestDataLE[8*4:], ZstdChunkedFrameMagic)
footer := ZstdChunkedFooterData{
ManifestType: uint64(ManifestTypeCRFS),
Offset: manifestOffset,
LengthCompressed: uint64(len(compressedManifest)),
LengthUncompressed: uint64(len(manifest)),
ChecksumAnnotation: "", // unused
OffsetTarSplit: uint64(tarSplitOffset),
LengthCompressedTarSplit: uint64(len(tarSplitData.Data)),
LengthUncompressedTarSplit: uint64(tarSplitData.UncompressedSize),
ChecksumAnnotationTarSplit: "", // unused
}
manifestDataLE := footerDataToBlob(footer)
return appendZstdSkippableFrame(dest, manifestDataLE)
}
@ -215,6 +221,21 @@ type ZstdChunkedFooterData struct {
ChecksumAnnotationTarSplit string // Only used when reading a layer, not when creating it
}
func footerDataToBlob(footer ZstdChunkedFooterData) []byte {
// Store the offset to the manifest and its size in LE order
manifestDataLE := make([]byte, FooterSizeSupported)
binary.LittleEndian.PutUint64(manifestDataLE[8*0:], footer.Offset)
binary.LittleEndian.PutUint64(manifestDataLE[8*1:], footer.LengthCompressed)
binary.LittleEndian.PutUint64(manifestDataLE[8*2:], footer.LengthUncompressed)
binary.LittleEndian.PutUint64(manifestDataLE[8*3:], footer.ManifestType)
binary.LittleEndian.PutUint64(manifestDataLE[8*4:], footer.OffsetTarSplit)
binary.LittleEndian.PutUint64(manifestDataLE[8*5:], footer.LengthCompressedTarSplit)
binary.LittleEndian.PutUint64(manifestDataLE[8*6:], footer.LengthUncompressedTarSplit)
copy(manifestDataLE[8*7:], ZstdChunkedFrameMagic)
return manifestDataLE
}
// ReadFooterDataFromAnnotations reads the zstd:chunked footer data from the given annotations.
func ReadFooterDataFromAnnotations(annotations map[string]string) (ZstdChunkedFooterData, error) {
var footerData ZstdChunkedFooterData
@ -239,14 +260,6 @@ func ReadFooterDataFromAnnotations(annotations map[string]string) (ZstdChunkedFo
return footerData, nil
}
// IsZstdChunkedFrameMagic returns true if the given data starts with the zstd:chunked magic.
func IsZstdChunkedFrameMagic(data []byte) bool {
if len(data) < 8 {
return false
}
return bytes.Equal(ZstdChunkedFrameMagic, data[:8])
}
// ReadFooterDataFromBlob reads the zstd:chunked footer from the binary buffer.
func ReadFooterDataFromBlob(footer []byte) (ZstdChunkedFooterData, error) {
var footerData ZstdChunkedFooterData
@ -258,7 +271,12 @@ func ReadFooterDataFromBlob(footer []byte) (ZstdChunkedFooterData, error) {
footerData.LengthCompressed = binary.LittleEndian.Uint64(footer[8:16])
footerData.LengthUncompressed = binary.LittleEndian.Uint64(footer[16:24])
footerData.ManifestType = binary.LittleEndian.Uint64(footer[24:32])
if !IsZstdChunkedFrameMagic(footer[48:56]) {
footerData.OffsetTarSplit = binary.LittleEndian.Uint64(footer[32:40])
footerData.LengthCompressedTarSplit = binary.LittleEndian.Uint64(footer[40:48])
footerData.LengthUncompressedTarSplit = binary.LittleEndian.Uint64(footer[48:56])
// the magic number is stored in the last 8 bytes
if !bytes.Equal(ZstdChunkedFrameMagic, footer[len(footer)-len(ZstdChunkedFrameMagic):]) {
return footerData, errors.New("invalid magic number")
}
return footerData, nil

29
storage/pkg/chunked/internal/compression_test.go
View File

@ -5,16 +5,29 @@ package internal
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestIsZstdChunkedFrameMagic(t *testing.T) {
b := append(ZstdChunkedFrameMagic[:], make([]byte, 200)...)
if !IsZstdChunkedFrameMagic(b) {
t.Fatal("Chunked frame magic not found")
func TestGenerateAndReadFooter(t *testing.T) {
footer := ZstdChunkedFooterData{
ManifestType: 1,
Offset: 2,
LengthCompressed: 3,
LengthUncompressed: 4,
ChecksumAnnotation: "", // unused
OffsetTarSplit: 5,
LengthCompressedTarSplit: 6,
LengthUncompressedTarSplit: 7,
ChecksumAnnotationTarSplit: "", // unused
}
// change a byte
b[0] = -b[0]
if IsZstdChunkedFrameMagic(b) {
t.Fatal("Invalid chunked frame magic found")
b := footerDataToBlob(footer)
assert.Len(t, b, FooterSizeSupported)
footer2, err := ReadFooterDataFromBlob(b)
if err != nil {
t.Fatal(err)
}
assert.Equal(t, footer, footer2)
}