containers
/
automation-tests
mirror of https://github.com/containers/automation-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1279 from rhatdan/digest 

Allow users to filter by digest
This commit is contained in:
OpenShift Merge Robot 2023-01-05 10:28:32 -05:00 committed by GitHub
parent 6ebae2a073 75bf2631d3
commit 480dfbe653
85 changed files with 1179 additions and 2011 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
common/go.mod
View File

@ -10,7 +10,7 @@ require (
github.com/containernetworking/plugins v1.1.1
github.com/containers/image/v5 v5.23.1-0.20221209092225-431fd251c4c5
github.com/containers/ocicrypt v1.1.6
github.com/containers/storage v1.44.1-0.20221209084436-73d739442168
github.com/containers/storage v1.44.1-0.20230105105526-fc91849352e5
github.com/coreos/go-systemd/v22 v22.5.0
github.com/cyphar/filepath-securejoin v0.2.3
github.com/davecgh/go-spew v1.1.1
@ -72,7 +72,7 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/klauspost/compress v1.15.12 // indirect
github.com/klauspost/compress v1.15.14 // indirect
github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf // indirect
@ -94,7 +94,7 @@ require (
github.com/tchap/go-patricia v2.3.0+incompatible // indirect
github.com/theupdateframework/go-tuf v0.5.2-0.20220930112810-3890c1e7ace4 // indirect
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/vbauerster/mpb/v7 v7.5.3 // indirect
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect

1337
common/go.sum
View File

File diff suppressed because it is too large Load Diff

10
common/libimage/filters.go
View File

@ -146,6 +146,9 @@ func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOp
case "id":
filter = filterID(value)
case "digest":
filter = filterDigest(value)
case "intermediate":
intermediate, err := r.bool(duplicate, key, value)
if err != nil {
@ -383,6 +386,13 @@ func filterID(value string) filterFunc {
}
}
// filterDigest creates an digest filter for matching the specified value.
func filterDigest(value string) filterFunc {
return func(img *Image) (bool, error) {
return string(img.Digest()) == value, nil
}
}
// filterIntermediate creates an intermediate filter for images. An image is
// considered to be an intermediate image if it is dangling (i.e., no tags) and
// has no children (i.e., no other image depends on it).

39
common/libimage/filters_test.go
View File

@ -72,6 +72,45 @@ func TestFilterReference(t *testing.T) {
}
}
func TestFilterDigest(t *testing.T) {
busyboxLatest := "quay.io/libpod/busybox:latest"
alpineLatest := "quay.io/libpod/alpine:latest"
runtime, cleanup := testNewRuntime(t)
defer cleanup()
ctx := context.Background()
pullOptions := &PullOptions{}
pullOptions.Writer = os.Stdout
pulledImages, err := runtime.Pull(ctx, busyboxLatest, config.PullPolicyMissing, pullOptions)
require.NoError(t, err)
require.Len(t, pulledImages, 1)
busybox := pulledImages[0]
pulledImages, err = runtime.Pull(ctx, alpineLatest, config.PullPolicyMissing, pullOptions)
require.NoError(t, err)
require.Len(t, pulledImages, 1)
alpine := pulledImages[0]
for _, test := range []struct {
filter string
matches int
id string
}{
{string(busybox.Digest()), 1, busybox.ID()},
{string(alpine.Digest()), 1, alpine.ID()},
} {
listOptions := &ListImagesOptions{
Filters: []string{"digest=" + test.filter},
}
listedImages, err := runtime.ListImages(ctx, nil, listOptions)
require.NoError(t, err, "%v", test)
require.Len(t, listedImages, test.matches, "%s -> %v", test.filter, listedImages)
require.Equal(t, listedImages[0].ID(), test.id)
}
}
func TestFilterManifest(t *testing.T) {
busyboxLatest := "quay.io/libpod/busybox:latest"
alpineLatest := "quay.io/libpod/alpine:latest"

180
common/vendor/github.com/containers/storage/containers.go generated vendored
View File

@ -141,16 +141,21 @@ type rwContainerStore interface {
}
type containerStore struct {
lockfile *lockfile.LockFile
dir string
jsonPath [numContainerLocationIndex]string
// The following fields are only set when constructing containerStore, and must never be modified afterwards.
// They are safe to access without any other locking.
lockfile *lockfile.LockFile // Synchronizes readers vs. writers of the _filesystem data_, both cross-process and in-process.
dir string
jsonPath [numContainerLocationIndex]string
inProcessLock sync.RWMutex // Can _only_ be obtained with lockfile held.
// The following fields can only be read/written with read/write ownership of inProcessLock, respectively.
// Almost all users should use startReading() or startWriting().
lastWrite lockfile.LastWrite
containers []*Container
idindex *truncindex.TruncIndex
byid map[string]*Container
bylayer map[string]*Container
byname map[string]*Container
loadMut sync.Mutex
}
func copyContainer(c *Container) *Container {
@ -202,6 +207,7 @@ func (c *Container) MountOpts() []string {
}
}
// The caller must hold r.inProcessLock for reading.
func containerLocation(c *Container) containerLocations {
if c.volatileStore {
return volatileContainerLocation
@ -216,9 +222,11 @@ func containerLocation(c *Container) containerLocations {
// should use startWriting() instead.
func (r *containerStore) startWritingWithReload(canReload bool) error {
r.lockfile.Lock()
r.inProcessLock.Lock()
succeeded := false
defer func() {
if !succeeded {
r.inProcessLock.Unlock()
r.lockfile.Unlock()
}
}()
@ -241,71 +249,135 @@ func (r *containerStore) startWriting() error {
// stopWriting releases locks obtained by startWriting.
func (r *containerStore) stopWriting() {
r.inProcessLock.Unlock()
r.lockfile.Unlock()
}
// startReading makes sure the store is fresh, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
func (r *containerStore) startReading() error {
// inProcessLocked calls the nested function with r.inProcessLock held for writing.
inProcessLocked := func(fn func() error) error {
r.inProcessLock.Lock()
defer r.inProcessLock.Unlock()
return fn()
}
r.lockfile.RLock()
unlockFn := r.lockfile.Unlock // A function to call to clean up, or nil
unlockFn := r.lockfile.Unlock // A function to call to clean up, or nil.
defer func() {
if unlockFn != nil {
unlockFn()
}
}()
r.inProcessLock.RLock()
unlockFn = r.stopReading
if tryLockedForWriting, err := r.reloadIfChanged(false); err != nil {
if !tryLockedForWriting {
return err
}
unlockFn()
unlockFn = nil
r.lockfile.Lock()
unlockFn = r.lockfile.Unlock
if _, err := r.reloadIfChanged(true); err != nil {
return err
}
unlockFn()
unlockFn = nil
r.lockfile.RLock()
unlockFn = r.lockfile.Unlock
// We need to check for a reload reload once more because the on-disk state could have been modified
// after we released the lock.
// If that, _again_, finds inconsistent state, just give up.
// We could, plausibly, retry a few times, but that inconsistent state (duplicate container names)
// shouldn’t be saved (by correct implementations) in the first place.
if _, err := r.reloadIfChanged(false); err != nil {
return fmt.Errorf("(even after successfully cleaning up once:) %w", err)
}
// If we are lucky, we can just hold the read locks, check that we are fresh, and continue.
_, modified, err := r.modified()
if err != nil {
return err
}
if modified {
// We are unlucky, and need to reload.
// NOTE: Multiple goroutines can get to this place approximately simultaneously.
r.inProcessLock.RUnlock()
unlockFn = r.lockfile.Unlock
// r.lastWrite can change at this point if another goroutine reloads the store before us. That’s why we don’t unconditionally
// trigger a load below; we (lock and) reloadIfChanged() again.
// First try reloading with r.lockfile held for reading.
// r.inProcessLock will serialize all goroutines that got here;
// each will re-check on-disk state vs. r.lastWrite, and the first one will actually reload the data.
var tryLockedForWriting bool
if err := inProcessLocked(func() error {
// We could optimize this further: The r.lockfile.GetLastWrite() value shouldn’t change as long as we hold r.lockfile,
// so if r.lastWrite was already updated, we don’t need to actually read the on-filesystem lock.
var err error
tryLockedForWriting, err = r.reloadIfChanged(false)
return err
}); err != nil {
if !tryLockedForWriting {
return err
}
// Not good enough, we need r.lockfile held for writing. So, let’s do that.
unlockFn()
unlockFn = nil
r.lockfile.Lock()
unlockFn = r.lockfile.Unlock
if err := inProcessLocked(func() error {
_, err := r.reloadIfChanged(true)
return err
}); err != nil {
return err
}
unlockFn()
unlockFn = nil
r.lockfile.RLock()
unlockFn = r.lockfile.Unlock
// We need to check for a reload once more because the on-disk state could have been modified
// after we released the lock.
// If that, _again_, finds inconsistent state, just give up.
// We could, plausibly, retry a few times, but that inconsistent state (duplicate container names)
// shouldn’t be saved (by correct implementations) in the first place.
if err := inProcessLocked(func() error {
_, err := r.reloadIfChanged(false)
return err
}); err != nil {
return fmt.Errorf("(even after successfully cleaning up once:) %w", err)
}
}
// NOTE that we hold neither a read nor write inProcessLock at this point. That’s fine in ordinary operation, because
// the on-filesystem r.lockfile should protect us against (cooperating) writers, and any use of r.inProcessLock
// protects us against in-process writers modifying data.
// In presence of non-cooperating writers, we just ensure that 1) the in-memory data is not clearly out-of-date
// and 2) access to the in-memory data is not racy;
// but we can’t protect against those out-of-process writers modifying _files_ while we are assuming they are in a consistent state.
r.inProcessLock.RLock()
}
unlockFn = nil
return nil
}
// stopReading releases locks obtained by startReading.
func (r *containerStore) stopReading() {
r.inProcessLock.RUnlock()
r.lockfile.Unlock()
}
// modified returns true if the on-disk state has changed (i.e. if reloadIfChanged may need to modify the store),
// and a lockfile.LastWrite value for that update.
//
// The caller must hold r.lockfile for reading _or_ writing.
// The caller must hold r.inProcessLock for reading or writing.
func (r *containerStore) modified() (lockfile.LastWrite, bool, error) {
return r.lockfile.ModifiedSince(r.lastWrite)
}
// reloadIfChanged reloads the contents of the store from disk if it is changed.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
//
// The caller must hold r.inProcessLock for WRITING.
//
// If !lockedForWriting and this function fails, the return value indicates whether
// reloadIfChanged() with lockedForWriting could succeed.
func (r *containerStore) reloadIfChanged(lockedForWriting bool) (bool, error) {
r.loadMut.Lock()
defer r.loadMut.Unlock()
lastWrite, modified, err := r.lockfile.ModifiedSince(r.lastWrite)
lastWrite, modified, err := r.modified()
if err != nil {
return false, err
}
// We require callers to always hold r.inProcessLock for WRITING, even if they might not end up calling r.load()
// and modify no fields, to ensure they see fresh data:
// r.lockfile.Modified() only returns true once per change. Without an exclusive lock,
// one goroutine might see r.lockfile.Modified() == true and decide to load, and in the meanwhile another one could
// see r.lockfile.Modified() == false and proceed to use in-memory data without noticing it is stale.
if modified {
if tryLockedForWriting, err := r.load(lockedForWriting); err != nil {
return tryLockedForWriting, err // r.lastWrite is unchanged, so we will load the next time again.
@ -315,6 +387,7 @@ func (r *containerStore) reloadIfChanged(lockedForWriting bool) (bool, error) {
return false, nil
}
// Requires startReading or startWriting.
func (r *containerStore) Containers() ([]Container, error) {
containers := make([]Container, len(r.containers))
for i := range r.containers {
@ -326,6 +399,7 @@ func (r *containerStore) Containers() ([]Container, error) {
// This looks for datadirs in the store directory that are not referenced
// by the json file and removes it. These can happen in the case of unclean
// shutdowns or regular restarts in transient store mode.
// Requires startReading.
func (r *containerStore) GarbageCollect() error {
entries, err := os.ReadDir(r.dir)
if err != nil {
@ -371,6 +445,7 @@ func (r *containerStore) datapath(id, key string) string {
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
// The caller must hold r.inProcessLock for WRITING.
//
// If !lockedForWriting and this function fails, the return value indicates whether
// retrying with lockedForWriting could succeed.
@ -441,8 +516,9 @@ func (r *containerStore) load(lockedForWriting bool) (bool, error) {
return false, nil
}
// Save saves the contents of the store to disk. It should be called with
// the lock held, locked for writing.
// save saves the contents of the store to disk.
// The caller must hold r.lockfile locked for writing.
// The caller must hold r.inProcessLock for reading (but usually holds it for writing in order to make the desired changes).
func (r *containerStore) save(saveLocations containerLocations) error {
r.lockfile.AssertLockedForWriting()
for locationIndex := 0; locationIndex < numContainerLocationIndex; locationIndex++ {
@ -483,6 +559,9 @@ func (r *containerStore) save(saveLocations containerLocations) error {
return nil
}
// saveFor saves the contents of the store relevant for modifiedContainer to disk.
// The caller must hold r.lockfile locked for writing.
// The caller must hold r.inProcessLock for reading (but usually holds it for writing in order to make the desired changes).
func (r *containerStore) saveFor(modifiedContainer *Container) error {
return r.save(containerLocation(modifiedContainer))
}
@ -503,16 +582,17 @@ func newContainerStore(dir string, runDir string, transient bool) (rwContainerSt
return nil, err
}
cstore := containerStore{
lockfile: lockfile,
dir: dir,
containers: []*Container{},
byid: make(map[string]*Container),
bylayer: make(map[string]*Container),
byname: make(map[string]*Container),
lockfile: lockfile,
dir: dir,
jsonPath: [numContainerLocationIndex]string{
filepath.Join(dir, "containers.json"),
filepath.Join(volatileDir, "volatile-containers.json"),
},
containers: []*Container{},
byid: make(map[string]*Container),
bylayer: make(map[string]*Container),
byname: make(map[string]*Container),
}
if err := cstore.startWritingWithReload(false); err != nil {
@ -529,6 +609,7 @@ func newContainerStore(dir string, runDir string, transient bool) (rwContainerSt
return &cstore, nil
}
// Requires startReading or startWriting.
func (r *containerStore) lookup(id string) (*Container, bool) {
if container, ok := r.byid[id]; ok {
return container, ok
@ -544,6 +625,7 @@ func (r *containerStore) lookup(id string) (*Container, bool) {
return nil, false
}
// Requires startWriting.
func (r *containerStore) ClearFlag(id string, flag string) error {
container, ok := r.lookup(id)
if !ok {
@ -553,6 +635,7 @@ func (r *containerStore) ClearFlag(id string, flag string) error {
return r.saveFor(container)
}
// Requires startWriting.
func (r *containerStore) SetFlag(id string, flag string, value interface{}) error {
container, ok := r.lookup(id)
if !ok {
@ -565,6 +648,7 @@ func (r *containerStore) SetFlag(id string, flag string, value interface{}) erro
return r.saveFor(container)
}
// Requires startWriting.
func (r *containerStore) Create(id string, names []string, image, layer, metadata string, options *ContainerOptions) (container *Container, err error) {
if id == "" {
id = stringid.GenerateRandomID()
@ -624,6 +708,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
return container, err
}
// Requires startReading or startWriting.
func (r *containerStore) Metadata(id string) (string, error) {
if container, ok := r.lookup(id); ok {
return container.Metadata, nil
@ -631,6 +716,7 @@ func (r *containerStore) Metadata(id string) (string, error) {
return "", ErrContainerUnknown
}
// Requires startWriting.
func (r *containerStore) SetMetadata(id, metadata string) error {
if container, ok := r.lookup(id); ok {
container.Metadata = metadata
@ -639,10 +725,12 @@ func (r *containerStore) SetMetadata(id, metadata string) error {
return ErrContainerUnknown
}
// The caller must hold r.inProcessLock for writing.
func (r *containerStore) removeName(container *Container, name string) {
container.Names = stringSliceWithoutValue(container.Names, name)
}
// Requires startWriting.
func (r *containerStore) updateNames(id string, names []string, op updateNameOperation) error {
container, ok := r.lookup(id)
if !ok {
@ -666,6 +754,7 @@ func (r *containerStore) updateNames(id string, names []string, op updateNameOpe
return r.saveFor(container)
}
// Requires startWriting.
func (r *containerStore) Delete(id string) error {
container, ok := r.lookup(id)
if !ok {
@ -704,6 +793,7 @@ func (r *containerStore) Delete(id string) error {
return nil
}
// Requires startReading or startWriting.
func (r *containerStore) Get(id string) (*Container, error) {
if container, ok := r.lookup(id); ok {
return copyContainer(container), nil
@ -711,6 +801,7 @@ func (r *containerStore) Get(id string) (*Container, error) {
return nil, ErrContainerUnknown
}
// Requires startReading or startWriting.
func (r *containerStore) Lookup(name string) (id string, err error) {
if container, ok := r.lookup(name); ok {
return container.ID, nil
@ -718,11 +809,13 @@ func (r *containerStore) Lookup(name string) (id string, err error) {
return "", ErrContainerUnknown
}
// Requires startReading or startWriting.
func (r *containerStore) Exists(id string) bool {
_, ok := r.lookup(id)
return ok
}
// Requires startReading or startWriting.
func (r *containerStore) BigData(id, key string) ([]byte, error) {
if key == "" {
return nil, fmt.Errorf("can't retrieve container big data value for empty name: %w", ErrInvalidBigDataName)
@ -790,6 +883,7 @@ func (r *containerStore) BigDataDigest(id, key string) (digest.Digest, error) {
return "", ErrDigestUnknown
}
// Requires startReading or startWriting.
func (r *containerStore) BigDataNames(id string) ([]string, error) {
c, ok := r.lookup(id)
if !ok {
@ -798,6 +892,7 @@ func (r *containerStore) BigDataNames(id string) ([]string, error) {
return copyStringSlice(c.BigDataNames), nil
}
// Requires startWriting.
func (r *containerStore) SetBigData(id, key string, data []byte) error {
if key == "" {
return fmt.Errorf("can't set empty name for container big data item: %w", ErrInvalidBigDataName)
@ -844,6 +939,7 @@ func (r *containerStore) SetBigData(id, key string, data []byte) error {
return err
}
// Requires startWriting.
func (r *containerStore) Wipe() error {
ids := make([]string, 0, len(r.byid))
for id := range r.byid {

5
common/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go generated vendored
View File

@ -6,6 +6,9 @@ package btrfs
/*
#include <stdlib.h>
#include <dirent.h>
// keep struct field name compatible with btrfs-progs < 6.1.
#define max_referenced max_rfer
#include <btrfs/ioctl.h>
#include <btrfs/ctree.h>
@ -382,7 +385,7 @@ func subvolLimitQgroup(path string, size uint64) error {
defer closeDir(dir)
var args C.struct_btrfs_ioctl_qgroup_limit_args
args.lim.max_referenced = C.__u64(size)
args.lim.max_rfer = C.__u64(size)
args.lim.flags = C.BTRFS_QGROUP_LIMIT_MAX_RFER
_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_LIMIT,
uintptr(unsafe.Pointer(&args)))

5
common/vendor/github.com/containers/storage/drivers/counter.go generated vendored
View File

@ -58,6 +58,11 @@ func (c *RefCounter) incdec(path string, infoOp func(minfo *minfo)) int {
}
infoOp(m)
count := m.count
if count <= 0 {
// If the mounted path has been decremented enough have no references,
// then its entry can be removed.
delete(c.counts, path)
}
c.mu.Unlock()
return count
}

3
common/vendor/github.com/containers/storage/drivers/overlay/overlay.go generated vendored
View File

@ -1202,6 +1202,9 @@ func (d *Driver) Remove(id string) error {
if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) {
return err
}
if d.quotaCtl != nil {
d.quotaCtl.ClearQuota(dir)
}
return nil
}

6
common/vendor/github.com/containers/storage/drivers/quota/projectquota.go generated vendored
View File

@ -211,6 +211,12 @@ func (q *Control) SetQuota(targetPath string, quota Quota) error {
return q.setProjectQuota(projectID, quota)
}
// ClearQuota removes the map entry in the quotas map for targetPath.
// It does so to prevent the map leaking entries as directories are deleted.
func (q *Control) ClearQuota(targetPath string) {
delete(q.quotas, targetPath)
}
// setProjectQuota - set the quota for project id on xfs block device
func (q *Control) setProjectQuota(projectID uint32, quota Quota) error {
var d C.fs_disk_quota_t

4
common/vendor/github.com/containers/storage/drivers/zfs/zfs.go generated vendored
View File

@ -57,12 +57,12 @@ func Init(base string, opt graphdriver.Options) (graphdriver.Driver, error) {
return nil, fmt.Errorf("the 'zfs' command is not available: %w", graphdriver.ErrPrerequisites)
}
file, err := os.OpenFile("/dev/zfs", os.O_RDWR, 0600)
file, err := unix.Open("/dev/zfs", unix.O_RDWR, 0600)
if err != nil {
logger.Debugf("cannot open /dev/zfs: %v", err)
return nil, fmt.Errorf("could not open /dev/zfs: %v: %w", err, graphdriver.ErrPrerequisites)
}
defer file.Close()
defer unix.Close(file)
options, err := parseOptions(opt.DriverOptions)
if err != nil {

4
common/vendor/github.com/containers/storage/idset.go generated vendored
View File

@ -1,11 +1,11 @@
package storage
import (
"errors"
"fmt"
"strings"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/types"
"github.com/google/go-intervals/intervalset"
)
@ -116,7 +116,7 @@ func (s *idSet) findAvailable(n int) (*idSet, error) {
n -= i.length()
}
if n > 0 {
return nil, errors.New("could not find enough available IDs")
return nil, types.ErrNoAvailableIDs
}
return &idSet{set: intervalset.NewImmutableSet(intervals)}, nil
}

161
common/vendor/github.com/containers/storage/images.go generated vendored
View File

@ -157,15 +157,20 @@ type rwImageStore interface {
}
type imageStore struct {
lockfile *lockfile.LockFile // lockfile.IsReadWrite can be used to distinguish between read-write and read-only image stores.
dir string
// The following fields are only set when constructing imageStore, and must never be modified afterwards.
// They are safe to access without any other locking.
lockfile *lockfile.LockFile // lockfile.IsReadWrite can be used to distinguish between read-write and read-only image stores.
dir string
inProcessLock sync.RWMutex // Can _only_ be obtained with lockfile held.
// The following fields can only be read/written with read/write ownership of inProcessLock, respectively.
// Almost all users should use startReading() or startWriting().
lastWrite lockfile.LastWrite
images []*Image
idindex *truncindex.TruncIndex
byid map[string]*Image
byname map[string]*Image
bydigest map[digest.Digest][]*Image
loadMut sync.Mutex
}
func copyImage(i *Image) *Image {
@ -205,9 +210,11 @@ func copyImageSlice(slice []*Image) []*Image {
// should use startReading() instead.
func (r *imageStore) startWritingWithReload(canReload bool) error {
r.lockfile.Lock()
r.inProcessLock.Lock()
succeeded := false
defer func() {
if !succeeded {
r.inProcessLock.Unlock()
r.lockfile.Unlock()
}
}()
@ -230,6 +237,7 @@ func (r *imageStore) startWriting() error {
// stopWriting releases locks obtained by startWriting.
func (r *imageStore) stopWriting() {
r.inProcessLock.Unlock()
r.lockfile.Unlock()
}
@ -239,6 +247,13 @@ func (r *imageStore) stopWriting() {
// This is an internal implementation detail of imageStore construction, every other caller
// should use startReading() instead.
func (r *imageStore) startReadingWithReload(canReload bool) error {
// inProcessLocked calls the nested function with r.inProcessLock held for writing.
inProcessLocked := func(fn func() error) error {
r.inProcessLock.Lock()
defer r.inProcessLock.Unlock()
return fn()
}
r.lockfile.RLock()
unlockFn := r.lockfile.Unlock // A function to call to clean up, or nil
defer func() {
@ -246,33 +261,76 @@ func (r *imageStore) startReadingWithReload(canReload bool) error {
unlockFn()
}
}()
r.inProcessLock.RLock()
unlockFn = r.stopReading
if canReload {
if tryLockedForWriting, err := r.reloadIfChanged(false); err != nil {
if !tryLockedForWriting {
return err
}
unlockFn()
unlockFn = nil
r.lockfile.Lock()
// If we are lucky, we can just hold the read locks, check that we are fresh, and continue.
_, modified, err := r.modified()
if err != nil {
return err
}
if modified {
// We are unlucky, and need to reload.
// NOTE: Multiple goroutines can get to this place approximately simultaneously.
r.inProcessLock.RUnlock()
unlockFn = r.lockfile.Unlock
if _, err := r.reloadIfChanged(true); err != nil {
return err
}
unlockFn()
unlockFn = nil
r.lockfile.RLock()
unlockFn = r.lockfile.Unlock
// We need to check for a reload reload once more because the on-disk state could have been modified
// after we released the lock.
// If that, _again_, finds inconsistent state, just give up.
// We could, plausibly, retry a few times, but that inconsistent state (duplicate image names)
// shouldn’t be saved (by correct implementations) in the first place.
if _, err := r.reloadIfChanged(false); err != nil {
return fmt.Errorf("(even after successfully cleaning up once:) %w", err)
// r.lastWrite can change at this point if another goroutine reloads the store before us. That’s why we don’t unconditionally
// trigger a load below; we (lock and) reloadIfChanged() again.
// First try reloading with r.lockfile held for reading.
// r.inProcessLock will serialize all goroutines that got here;
// each will re-check on-disk state vs. r.lastWrite, and the first one will actually reload the data.
var tryLockedForWriting bool
if err := inProcessLocked(func() error {
// We could optimize this further: The r.lockfile.GetLastWrite() value shouldn’t change as long as we hold r.lockfile,
// so if r.lastWrite was already updated, we don’t need to actually read the on-filesystem lock.
var err error
tryLockedForWriting, err = r.reloadIfChanged(false)
return err
}); err != nil {
if !tryLockedForWriting {
return err
}
// Not good enough, we need r.lockfile held for writing. So, let’s do that.
unlockFn()
unlockFn = nil
r.lockfile.Lock()
unlockFn = r.lockfile.Unlock
if err := inProcessLocked(func() error {
_, err := r.reloadIfChanged(true)
return err
}); err != nil {
return err
}
unlockFn()
unlockFn = nil
r.lockfile.RLock()
unlockFn = r.lockfile.Unlock
// We need to check for a reload once more because the on-disk state could have been modified
// after we released the lock.
// If that, _again_, finds inconsistent state, just give up.
// We could, plausibly, retry a few times, but that inconsistent state (duplicate image names)
// shouldn’t be saved (by correct implementations) in the first place.
if err := inProcessLocked(func() error {
_, err := r.reloadIfChanged(false)
return err
}); err != nil {
return fmt.Errorf("(even after successfully cleaning up once:) %w", err)
}
}
// NOTE that we hold neither a read nor write inProcessLock at this point. That’s fine in ordinary operation, because
// the on-filesystem r.lockfile should protect us against (cooperating) writers, and any use of r.inProcessLock
// protects us against in-process writers modifying data.
// In presence of non-cooperating writers, we just ensure that 1) the in-memory data is not clearly out-of-date
// and 2) access to the in-memory data is not racy;
// but we can’t protect against those out-of-process writers modifying _files_ while we are assuming they are in a consistent state.
r.inProcessLock.RLock()
}
}
@ -288,24 +346,38 @@ func (r *imageStore) startReading() error {
// stopReading releases locks obtained by startReading.
func (r *imageStore) stopReading() {
r.inProcessLock.RUnlock()
r.lockfile.Unlock()
}
// modified returns true if the on-disk state has changed (i.e. if reloadIfChanged may need to modify the store),
// and a lockfile.LastWrite value for that update.
//
// The caller must hold r.lockfile for reading _or_ writing.
// The caller must hold r.inProcessLock for reading or writing.
func (r *imageStore) modified() (lockfile.LastWrite, bool, error) {
return r.lockfile.ModifiedSince(r.lastWrite)
}
// reloadIfChanged reloads the contents of the store from disk if it is changed.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
//
// The caller must hold r.inProcessLock for WRITING.
//
// If !lockedForWriting and this function fails, the return value indicates whether
// reloadIfChanged() with lockedForWriting could succeed.
func (r *imageStore) reloadIfChanged(lockedForWriting bool) (bool, error) {
r.loadMut.Lock()
defer r.loadMut.Unlock()
lastWrite, modified, err := r.lockfile.ModifiedSince(r.lastWrite)
lastWrite, modified, err := r.modified()
if err != nil {
return false, err
}
// We require callers to always hold r.inProcessLock for WRITING, even if they might not end up calling r.load()
// and modify no fields, to ensure they see fresh data:
// r.lockfile.Modified() only returns true once per change. Without an exclusive lock,
// one goroutine might see r.lockfile.Modified() == true and decide to load, and in the meanwhile another one could
// see r.lockfile.Modified() == false and proceed to use in-memory data without noticing it is stale.
if modified {
if tryLockedForWriting, err := r.load(lockedForWriting); err != nil {
return tryLockedForWriting, err // r.lastWrite is unchanged, so we will load the next time again.
@ -315,6 +387,7 @@ func (r *imageStore) reloadIfChanged(lockedForWriting bool) (bool, error) {
return false, nil
}
// Requires startReading or startWriting.
func (r *imageStore) Images() ([]Image, error) {
images := make([]Image, len(r.images))
for i := range r.images {
@ -345,6 +418,7 @@ func bigDataNameIsManifest(name string) bool {
// recomputeDigests takes a fixed digest and a name-to-digest map and builds a
// list of the unique values that would identify the image.
// The caller must hold r.inProcessLock for writing.
func (i *Image) recomputeDigests() error {
validDigests := make([]digest.Digest, 0, len(i.BigDataDigests)+1)
digests := make(map[digest.Digest]struct{})
@ -382,6 +456,7 @@ func (i *Image) recomputeDigests() error {
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
// The caller must hold r.inProcessLock for WRITING.
//
// If !lockedForWriting and this function fails, the return value indicates whether
// retrying with lockedForWriting could succeed.
@ -445,8 +520,9 @@ func (r *imageStore) load(lockedForWriting bool) (bool, error) {
return false, nil
}
// Save saves the contents of the store to disk. It should be called with
// the lock held, locked for writing.
// Save saves the contents of the store to disk.
// The caller must hold r.lockfile locked for writing.
// The caller must hold r.inProcessLock for reading (but usually holds it for writing in order to make the desired changes).
func (r *imageStore) Save() error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify the image store at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
@ -482,6 +558,7 @@ func newImageStore(dir string) (rwImageStore, error) {
istore := imageStore{
lockfile: lockfile,
dir: dir,
images: []*Image{},
byid: make(map[string]*Image),
byname: make(map[string]*Image),
@ -509,6 +586,7 @@ func newROImageStore(dir string) (roImageStore, error) {
istore := imageStore{
lockfile: lockfile,
dir: dir,
images: []*Image{},
byid: make(map[string]*Image),
byname: make(map[string]*Image),
@ -528,6 +606,7 @@ func newROImageStore(dir string) (roImageStore, error) {
return &istore, nil
}
// Requires startReading or startWriting.
func (r *imageStore) lookup(id string) (*Image, bool) {
if image, ok := r.byid[id]; ok {
return image, ok
@ -540,6 +619,7 @@ func (r *imageStore) lookup(id string) (*Image, bool) {
return nil, false
}
// Requires startWriting.
func (r *imageStore) ClearFlag(id string, flag string) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to clear flags on images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
@ -552,6 +632,7 @@ func (r *imageStore) ClearFlag(id string, flag string) error {
return r.Save()
}
// Requires startWriting.
func (r *imageStore) SetFlag(id string, flag string, value interface{}) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to set flags on images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
@ -567,6 +648,7 @@ func (r *imageStore) SetFlag(id string, flag string, value interface{}) error {
return r.Save()
}
// Requires startWriting.
func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (image *Image, err error) {
if !r.lockfile.IsReadWrite() {
return nil, fmt.Errorf("not allowed to create new images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
@ -626,6 +708,7 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string, c
return image, err
}
// Requires startWriting.
func (r *imageStore) addMappedTopLayer(id, layer string) error {
if image, ok := r.lookup(id); ok {
image.MappedTopLayers = append(image.MappedTopLayers, layer)
@ -634,6 +717,7 @@ func (r *imageStore) addMappedTopLayer(id, layer string) error {
return fmt.Errorf("locating image with ID %q: %w", id, ErrImageUnknown)
}
// Requires startWriting.
func (r *imageStore) removeMappedTopLayer(id, layer string) error {
if image, ok := r.lookup(id); ok {
initialLen := len(image.MappedTopLayers)
@ -647,6 +731,7 @@ func (r *imageStore) removeMappedTopLayer(id, layer string) error {
return fmt.Errorf("locating image with ID %q: %w", id, ErrImageUnknown)
}
// Requires startReading or startWriting.
func (r *imageStore) Metadata(id string) (string, error) {
if image, ok := r.lookup(id); ok {
return image.Metadata, nil
@ -654,6 +739,7 @@ func (r *imageStore) Metadata(id string) (string, error) {
return "", fmt.Errorf("locating image with ID %q: %w", id, ErrImageUnknown)
}
// Requires startWriting.
func (r *imageStore) SetMetadata(id, metadata string) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify image metadata at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
@ -665,14 +751,17 @@ func (r *imageStore) SetMetadata(id, metadata string) error {
return fmt.Errorf("locating image with ID %q: %w", id, ErrImageUnknown)
}
// The caller must hold r.inProcessLock for writing.
func (r *imageStore) removeName(image *Image, name string) {
image.Names = stringSliceWithoutValue(image.Names, name)
}
// The caller must hold r.inProcessLock for writing.
func (i *Image) addNameToHistory(name string) {
i.NamesHistory = dedupeNames(append([]string{name}, i.NamesHistory...))
}
// Requires startWriting.
func (r *imageStore) updateNames(id string, names []string, op updateNameOperation) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to change image name assignments at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
@ -700,6 +789,7 @@ func (r *imageStore) updateNames(id string, names []string, op updateNameOperati
return r.Save()
}
// Requires startWriting.
func (r *imageStore) Delete(id string) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to delete images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
@ -747,6 +837,7 @@ func (r *imageStore) Delete(id string) error {
return nil
}
// Requires startReading or startWriting.
func (r *imageStore) Get(id string) (*Image, error) {
if image, ok := r.lookup(id); ok {
return copyImage(image), nil
@ -754,11 +845,13 @@ func (r *imageStore) Get(id string) (*Image, error) {
return nil, fmt.Errorf("locating image with ID %q: %w", id, ErrImageUnknown)
}
// Requires startReading or startWriting.
func (r *imageStore) Exists(id string) bool {
_, ok := r.lookup(id)
return ok
}
// Requires startReading or startWriting.
func (r *imageStore) ByDigest(d digest.Digest) ([]*Image, error) {
if images, ok := r.bydigest[d]; ok {
return copyImageSlice(images), nil
@ -766,6 +859,7 @@ func (r *imageStore) ByDigest(d digest.Digest) ([]*Image, error) {
return nil, fmt.Errorf("locating image with digest %q: %w", d, ErrImageUnknown)
}
// Requires startReading or startWriting.
func (r *imageStore) BigData(id, key string) ([]byte, error) {
if key == "" {
return nil, fmt.Errorf("can't retrieve image big data value for empty name: %w", ErrInvalidBigDataName)
@ -777,6 +871,7 @@ func (r *imageStore) BigData(id, key string) ([]byte, error) {
return os.ReadFile(r.datapath(image.ID, key))
}
// Requires startReading or startWriting.
func (r *imageStore) BigDataSize(id, key string) (int64, error) {
if key == "" {
return -1, fmt.Errorf("can't retrieve size of image big data with empty name: %w", ErrInvalidBigDataName)
@ -794,6 +889,7 @@ func (r *imageStore) BigDataSize(id, key string) (int64, error) {
return -1, ErrSizeUnknown
}
// Requires startReading or startWriting.
func (r *imageStore) BigDataDigest(id, key string) (digest.Digest, error) {
if key == "" {
return "", fmt.Errorf("can't retrieve digest of image big data value with empty name: %w", ErrInvalidBigDataName)
@ -808,6 +904,7 @@ func (r *imageStore) BigDataDigest(id, key string) (digest.Digest, error) {
return "", ErrDigestUnknown
}
// Requires startReading or startWriting.
func (r *imageStore) BigDataNames(id string) ([]string, error) {
image, ok := r.lookup(id)
if !ok {
@ -827,6 +924,7 @@ func imageSliceWithoutValue(slice []*Image, value *Image) []*Image {
return modified
}
// Requires startWriting.
func (r *imageStore) SetBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error {
if key == "" {
return fmt.Errorf("can't set empty name for image big data item: %w", ErrInvalidBigDataName)
@ -911,6 +1009,7 @@ func (r *imageStore) SetBigData(id, key string, data []byte, digestManifest func
return err
}
// Requires startWriting.
func (r *imageStore) Wipe() error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to delete images at %q: %w", r.imagespath(), ErrStoreIsReadOnly)

345
common/vendor/github.com/containers/storage/layers.go generated vendored
View File

@ -83,13 +83,26 @@ type Layer struct {
MountLabel string `json:"mountlabel,omitempty"`
// MountPoint is the path where the layer is mounted, or where it was most
// recently mounted. This can change between subsequent Unmount() and
// Mount() calls, so the caller should consult this value after Mount()
// succeeds to find the location of the container's root filesystem.
// recently mounted.
//
// WARNING: This field is a snapshot in time: (except for users inside c/storage that
// hold the mount lock) the true value can change between subsequent
// calls to c/storage API.
//
// Users that need to handle concurrent mount/unmount attempts should not access this
// field at all, and should only use the path returned by .Mount() (and that’s only
// assuming no other user will concurrently decide to unmount that mount point).
MountPoint string `json:"-"`
// MountCount is used as a reference count for the container's layer being
// mounted at the mount point.
//
// WARNING: This field is a snapshot in time; (except for users inside c/storage that
// hold the mount lock) the true value can change between subsequent
// calls to c/storage API.
//
// In situations where concurrent mount/unmount attempts can happen, this field
// should not be used for any decisions, maybe apart from heuristic user warnings.
MountCount int `json:"-"`
// Created is the datestamp for when this layer was created. Older
@ -265,8 +278,15 @@ type rwLayerStore interface {
// The mappings used by the container can be specified.
Mount(id string, options drivers.MountOpts) (string, error)
// Unmount unmounts a layer when it is no longer in use.
Unmount(id string, force bool) (bool, error)
// unmount unmounts a layer when it is no longer in use.
// If conditional is set, it will fail with ErrLayerNotMounted if the layer is not mounted (without conditional, the caller is
// making a promise that the layer is actually mounted).
// If force is set, it will physically try to unmount it even if it is mounted multple times, or even if (!conditional and)
// there are no records of it being mounted in the first place.
// It returns whether the layer was still mounted at the time this function returned.
// WARNING: The return value may already be obsolete by the time it is available
// to the caller, so it can be used for heuristic sanity checks at best. It should almost always be ignored.
unmount(id string, force bool, conditional bool) (bool, error)
// Mounted returns number of times the layer has been mounted.
Mounted(id string) (int, error)
@ -305,12 +325,17 @@ type rwLayerStore interface {
}
type layerStore struct {
lockfile *lockfile.LockFile
mountsLockfile *lockfile.LockFile
rundir string
jsonPath [numLayerLocationIndex]string
driver drivers.Driver
layerdir string
// The following fields are only set when constructing layerStore, and must never be modified afterwards.
// They are safe to access without any other locking.
lockfile *lockfile.LockFile // lockfile.IsReadWrite can be used to distinguish between read-write and read-only layer stores.
mountsLockfile *lockfile.LockFile // Can _only_ be obtained with inProcessLock held.
rundir string
jsonPath [numLayerLocationIndex]string
layerdir string
inProcessLock sync.RWMutex // Can _only_ be obtained with lockfile held.
// The following fields can only be read/written with read/write ownership of inProcessLock, respectively.
// Almost all users should use startReading() or startWriting().
lastWrite lockfile.LastWrite
mountsLastWrite lockfile.LastWrite // Only valid if lockfile.IsReadWrite()
layers []*Layer
@ -320,10 +345,14 @@ type layerStore struct {
bymount map[string]*Layer
bycompressedsum map[digest.Digest][]string
byuncompressedsum map[digest.Digest][]string
loadMut sync.Mutex
layerspathsModified [numLayerLocationIndex]time.Time
// FIXME: This field is only set when constructing layerStore, but locking rules of the driver
// interface itself are not documented here.
driver drivers.Driver
}
// The caller must hold r.inProcessLock for reading.
func layerLocation(l *Layer) layerLocations {
if l.volatileStore {
return volatileLayerLocation
@ -364,9 +393,11 @@ func copyLayer(l *Layer) *Layer {
// should use startWriting() instead.
func (r *layerStore) startWritingWithReload(canReload bool) error {
r.lockfile.Lock()
r.inProcessLock.Lock()
succeeded := false
defer func() {
if !succeeded {
r.inProcessLock.Unlock()
r.lockfile.Unlock()
}
}()
@ -389,6 +420,7 @@ func (r *layerStore) startWriting() error {
// stopWriting releases locks obtained by startWriting.
func (r *layerStore) stopWriting() {
r.inProcessLock.Unlock()
r.lockfile.Unlock()
}
@ -398,6 +430,13 @@ func (r *layerStore) stopWriting() {
// This is an internal implementation detail of layerStore construction, every other caller
// should use startReading() instead.
func (r *layerStore) startReadingWithReload(canReload bool) error {
// inProcessLocked calls the nested function with r.inProcessLock held for writing.
inProcessLocked := func(fn func() error) error {
r.inProcessLock.Lock()
defer r.inProcessLock.Unlock()
return fn()
}
r.lockfile.RLock()
unlockFn := r.lockfile.Unlock // A function to call to clean up, or nil
defer func() {
@ -405,36 +444,71 @@ func (r *layerStore) startReadingWithReload(canReload bool) error {
unlockFn()
}
}()
r.inProcessLock.RLock()
unlockFn = r.stopReading
if canReload {
cleanupsDone := 0
for {
tryLockedForWriting, err := r.reloadIfChanged(false)
if err == nil {
break
}
if !tryLockedForWriting {
return err
}
if cleanupsDone >= maxLayerStoreCleanupIterations {
return fmt.Errorf("(even after %d cleanup attempts:) %w", cleanupsDone, err)
}
unlockFn()
unlockFn = nil
r.lockfile.Lock()
// If we are lucky, we can just hold the read locks, check that we are fresh, and continue.
modified, err := r.modified()
if err != nil {
return err
}
if modified {
// We are unlucky, and need to reload.
// NOTE: Multiple goroutines can get to this place approximately simultaneously.
r.inProcessLock.RUnlock()
unlockFn = r.lockfile.Unlock
if _, err := r.reloadIfChanged(true); err != nil {
return err
}
unlockFn()
unlockFn = nil
r.lockfile.RLock()
unlockFn = r.lockfile.Unlock
// We need to check for a reload reload again because the on-disk state could have been modified
// after we released the lock.
cleanupsDone++
cleanupsDone := 0
for {
// First try reloading with r.lockfile held for reading.
// r.inProcessLock will serialize all goroutines that got here;
// each will re-check on-disk state vs. r.lastWrite, and the first one will actually reload the data.
var tryLockedForWriting bool
err := inProcessLocked(func() error {
var err error
tryLockedForWriting, err = r.reloadIfChanged(false)
return err
})
if err == nil {
break
}
if !tryLockedForWriting {
return err
}
if cleanupsDone >= maxLayerStoreCleanupIterations {
return fmt.Errorf("(even after %d cleanup attempts:) %w", cleanupsDone, err)
}
// Not good enough, we need r.lockfile held for writing. So, let’s do that.
unlockFn()
unlockFn = nil
r.lockfile.Lock()
unlockFn = r.lockfile.Unlock
if err := inProcessLocked(func() error {
_, err := r.reloadIfChanged(true)
return err
}); err != nil {
return err
}
unlockFn()
unlockFn = nil
r.lockfile.RLock()
unlockFn = r.lockfile.Unlock
// We need to check for a reload again because the on-disk state could have been modified
// after we released the lock.
cleanupsDone++
}
// NOTE that we hold neither a read nor write inProcessLock at this point. That’s fine in ordinary operation, because
// the on-filesystem r.lockfile should protect us against (cooperating) writers, and any use of r.inProcessLock
// protects us against in-process writers modifying data.
// In presence of non-cooperating writers, we just ensure that 1) the in-memory data is not clearly out-of-date
// and 2) access to the in-memory data is not racy;
// but we can’t protect against those out-of-process writers modifying _files_ while we are assuming they are in a consistent state.
r.inProcessLock.RLock()
}
}
@ -450,13 +524,48 @@ func (r *layerStore) startReading() error {
// stopReading releases locks obtained by startReading.
func (r *layerStore) stopReading() {
r.inProcessLock.RUnlock()
r.lockfile.Unlock()
}
// modified returns true if the on-disk state (of layers or mounts) has changed (ie if reloadIcHanged may need to modify the store)
//
// Note that unlike containerStore.modified and imageStore.modified, this function is not directly used in layerStore.reloadIfChanged();
// it exists only to help the reader ensure it has fresh enough state.
//
// The caller must hold r.lockfile for reading _or_ writing.
// The caller must hold r.inProcessLock for reading or writing.
func (r *layerStore) modified() (bool, error) {
_, m, err := r.layersModified()
if err != nil {
return false, err
}
if m {
return true, nil
}
if r.lockfile.IsReadWrite() {
// This means we get, release, and re-obtain, r.mountsLockfile if we actually need to do any kind of reload.
// That’s a bit expensive, but hopefully most callers will be read-only and see no changes.
// We can’t eliminate these mountsLockfile accesses given the current assumption that Layer objects have _some_ not-very-obsolete
// mount data. Maybe we can segregate the mount-dependent and mount-independent operations better...
r.mountsLockfile.RLock()
defer r.mountsLockfile.Unlock()
_, m, err := r.mountsModified()
if err != nil {
return false, err
}
if m {
return true, nil
}
}
return false, nil
}
// layersModified() checks if the most recent writer to r.jsonPath[] was a party other than the
// last recorded writer. If so, it returns a lockfile.LastWrite value to record on a successful
// reload.
// It should only be called with the lock held.
// The caller must hold r.inProcessLock for reading.
func (r *layerStore) layersModified() (lockfile.LastWrite, bool, error) {
lastWrite, modified, err := r.lockfile.ModifiedSince(r.lastWrite)
if err != nil {
@ -488,12 +597,11 @@ func (r *layerStore) layersModified() (lockfile.LastWrite, bool, error) {
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
//
// The caller must hold r.inProcessLock for WRITING.
//
// If !lockedForWriting and this function fails, the return value indicates whether
// reloadIfChanged() with lockedForWriting could succeed.
func (r *layerStore) reloadIfChanged(lockedForWriting bool) (bool, error) {
r.loadMut.Lock()
defer r.loadMut.Unlock()
lastWrite, layersModified, err := r.layersModified()
if err != nil {
return false, err
@ -516,11 +624,20 @@ func (r *layerStore) reloadIfChanged(lockedForWriting bool) (bool, error) {
return false, nil
}
// mountsModified returns true if the on-disk mount state has changed (i.e. if reloadMountsIfChanged may need to modify the store),
// and a lockfile.LastWrite value for that update.
//
// The caller must hold r.mountsLockfile for reading _or_ writing.
// The caller must hold r.inProcessLock for reading or writing.
func (r *layerStore) mountsModified() (lockfile.LastWrite, bool, error) {
return r.mountsLockfile.ModifiedSince(r.mountsLastWrite)
}
// reloadMountsIfChanged reloads the contents of mountsPath from disk if it is changed.
//
// The caller must hold r.mountsLockFile for reading or writing.
func (r *layerStore) reloadMountsIfChanged() error {
lastWrite, modified, err := r.mountsLockfile.ModifiedSince(r.mountsLastWrite)
lastWrite, modified, err := r.mountsModified()
if err != nil {
return err
}
@ -533,6 +650,7 @@ func (r *layerStore) reloadMountsIfChanged() error {
return nil
}
// Requires startReading or startWriting.
func (r *layerStore) Layers() ([]Layer, error) {
layers := make([]Layer, len(r.layers))
for i := range r.layers {
@ -541,6 +659,7 @@ func (r *layerStore) Layers() ([]Layer, error) {
return layers, nil
}
// Requires startWriting.
func (r *layerStore) GarbageCollect() error {
layers, err := r.driver.ListLayers()
@ -581,6 +700,7 @@ func (r *layerStore) mountspath() string {
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
// The caller must hold r.inProcessLock for WRITING.
//
// If !lockedForWriting and this function fails, the return value indicates whether
// retrying with lockedForWriting could succeed.
@ -694,6 +814,11 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) {
return false, err
}
r.mountsLastWrite = mountsLastWrite
// NOTE: We will release mountsLockfile when this function returns, so unlike most of the layer data, the
// r.layers[].MountPoint, r.layers[].MountCount, and r.bymount values might not reflect
// true on-filesystem state already by the time this function returns.
// Code that needs the state to be accurate must lock r.mountsLockfile again,
// and possibly loadMounts() again.
}
if errorToResolveBySaving != nil {
@ -731,6 +856,8 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) {
return false, nil
}
// The caller must hold r.mountsLockfile for reading or writing.
// The caller must hold r.inProcessLock for WRITING.
func (r *layerStore) loadMounts() error {
mounts := make(map[string]*Layer)
mpath := r.mountspath()
@ -768,8 +895,9 @@ func (r *layerStore) loadMounts() error {
return err
}
// Save saves the contents of the store to disk. It should be called with
// the lock held, locked for writing.
// save saves the contents of the store to disk.
// The caller must hold r.lockfile locked for writing.
// The caller must hold r.inProcessLock for WRITING.
func (r *layerStore) save(saveLocations layerLocations) error {
r.mountsLockfile.Lock()
defer r.mountsLockfile.Unlock()
@ -779,10 +907,15 @@ func (r *layerStore) save(saveLocations layerLocations) error {
return r.saveMounts()
}
// saveFor saves the contents of the store relevant for modifiedLayer to disk.
// The caller must hold r.lockfile locked for writing.
// The caller must hold r.inProcessLock for WRITING.
func (r *layerStore) saveFor(modifiedLayer *Layer) error {
return r.save(layerLocation(modifiedLayer))
}
// The caller must hold r.lockfile locked for writing.
// The caller must hold r.inProcessLock for WRITING.
func (r *layerStore) saveLayers(saveLocations layerLocations) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -826,6 +959,8 @@ func (r *layerStore) saveLayers(saveLocations layerLocations) error {
return nil
}
// The caller must hold r.mountsLockfile for writing.
// The caller must hold r.inProcessLock for WRITING.
func (r *layerStore) saveMounts() error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -887,16 +1022,18 @@ func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Dri
rlstore := layerStore{
lockfile: lockFile,
mountsLockfile: mountsLockfile,
driver: driver,
rundir: rundir,
layerdir: layerdir,
byid: make(map[string]*Layer),
bymount: make(map[string]*Layer),
byname: make(map[string]*Layer),
jsonPath: [numLayerLocationIndex]string{
filepath.Join(layerdir, "layers.json"),
filepath.Join(volatileDir, "volatile-layers.json"),
},
layerdir: layerdir,
byid: make(map[string]*Layer),
byname: make(map[string]*Layer),
bymount: make(map[string]*Layer),
driver: driver,
}
if err := rlstore.startWritingWithReload(false); err != nil {
return nil, err
@ -922,16 +1059,18 @@ func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (roL
rlstore := layerStore{
lockfile: lockfile,
mountsLockfile: nil,
driver: driver,
rundir: rundir,
layerdir: layerdir,
byid: make(map[string]*Layer),
bymount: make(map[string]*Layer),
byname: make(map[string]*Layer),
jsonPath: [numLayerLocationIndex]string{
filepath.Join(layerdir, "layers.json"),
filepath.Join(layerdir, "volatile-layers.json"),
},
layerdir: layerdir,
byid: make(map[string]*Layer),
byname: make(map[string]*Layer),
bymount: make(map[string]*Layer),
driver: driver,
}
if err := rlstore.startReadingWithReload(false); err != nil {
return nil, err
@ -948,6 +1087,7 @@ func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (roL
return &rlstore, nil
}
// Requires startReading or startWriting.
func (r *layerStore) lookup(id string) (*Layer, bool) {
if layer, ok := r.byid[id]; ok {
return layer, ok
@ -960,6 +1100,7 @@ func (r *layerStore) lookup(id string) (*Layer, bool) {
return nil, false
}
// Requires startReading or startWriting.
func (r *layerStore) Size(name string) (int64, error) {
layer, ok := r.lookup(name)
if !ok {
@ -974,6 +1115,7 @@ func (r *layerStore) Size(name string) (int64, error) {
return -1, nil
}
// Requires startWriting.
func (r *layerStore) ClearFlag(id string, flag string) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to clear flags on layers at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -986,6 +1128,7 @@ func (r *layerStore) ClearFlag(id string, flag string) error {
return r.saveFor(layer)
}
// Requires startWriting.
func (r *layerStore) SetFlag(id string, flag string, value interface{}) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to set flags on layers at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -1005,6 +1148,7 @@ func (r *layerStore) Status() ([][2]string, error) {
return r.driver.Status(), nil
}
// Requires startWriting.
func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) {
if duplicateLayer, idInUse := r.byid[id]; idInUse {
return duplicateLayer, ErrDuplicateID
@ -1061,6 +1205,7 @@ func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []s
return copyLayer(layer), nil
}
// Requires startWriting.
func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (*Layer, int64, error) {
if !r.lockfile.IsReadWrite() {
return nil, -1, fmt.Errorf("not allowed to create new layers at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -1258,32 +1403,41 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab
return layer, size, err
}
// Requires startWriting.
func (r *layerStore) CreateWithFlags(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}) (layer *Layer, err error) {
layer, _, err = r.Put(id, parent, names, mountLabel, options, moreOptions, writeable, flags, nil)
return layer, err
}
// Requires startWriting.
func (r *layerStore) Create(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool) (layer *Layer, err error) {
return r.CreateWithFlags(id, parent, names, mountLabel, options, moreOptions, writeable, nil)
}
// Requires startReading or startWriting.
func (r *layerStore) Mounted(id string) (int, error) {
if !r.lockfile.IsReadWrite() {
return 0, fmt.Errorf("no mount information for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly)
}
r.mountsLockfile.RLock()
defer r.mountsLockfile.Unlock()
if err := r.reloadMountsIfChanged(); err != nil {
return 0, err
}
layer, ok := r.lookup(id)
if !ok {
return 0, ErrLayerUnknown
}
// NOTE: The caller of this function is not holding (currently cannot hold) r.mountsLockfile,
// so the data is necessarily obsolete by the time this function returns. So, we don’t even
// try to reload it in this function, we just rely on r.load() that happened during
// r.startReading() or r.startWriting().
return layer.MountCount, nil
}
// Requires startWriting.
func (r *layerStore) Mount(id string, options drivers.MountOpts) (string, error) {
// LOCKING BUG: This is reachable via store.Diff → layerStore.Diff → layerStore.newFileGetter
// (with btrfs and zfs graph drivers) holding layerStore only locked for reading, while it modifies
// - r.layers[].MountCount (directly and via loadMounts / saveMounts)
// - r.layers[].MountPoint (directly and via loadMounts / saveMounts)
// - r.bymount (via loadMounts / saveMounts)
// check whether options include ro option
hasReadOnlyOpt := func(opts []string) bool {
for _, item := range opts {
@ -1343,7 +1497,14 @@ func (r *layerStore) Mount(id string, options drivers.MountOpts) (string, error)
return mountpoint, err
}
func (r *layerStore) Unmount(id string, force bool) (bool, error) {
// Requires startWriting.
func (r *layerStore) unmount(id string, force bool, conditional bool) (bool, error) {
// LOCKING BUG: This is reachable via store.Diff → layerStore.Diff → layerStore.newFileGetter → simpleGetCloser.Close()
// (with btrfs and zfs graph drivers) holding layerStore only locked for reading, while it modifies
// - r.layers[].MountCount (directly and via loadMounts / saveMounts)
// - r.layers[].MountPoint (directly and via loadMounts / saveMounts)
// - r.bymount (via loadMounts / saveMounts)
if !r.lockfile.IsReadWrite() {
return false, fmt.Errorf("not allowed to update mount locations for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly)
}
@ -1360,6 +1521,9 @@ func (r *layerStore) Unmount(id string, force bool) (bool, error) {
}
layer = layerByMount
}
if conditional && layer.MountCount == 0 {
return false, ErrLayerNotMounted
}
if force {
layer.MountCount = 1
}
@ -1379,15 +1543,16 @@ func (r *layerStore) Unmount(id string, force bool) (bool, error) {
return true, err
}
// Requires startReading or startWriting.
func (r *layerStore) ParentOwners(id string) (uids, gids []int, err error) {
if !r.lockfile.IsReadWrite() {
return nil, nil, fmt.Errorf("no mount information for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly)
}
r.mountsLockfile.RLock()
defer r.mountsLockfile.Unlock()
if err := r.reloadMountsIfChanged(); err != nil {
return nil, nil, err
}
// We are not checking r.mountsLockfile.Modified() and calling r.loadMounts here because the store
// is only locked for reading = we are not allowed to modify layer data.
// Holding r.mountsLockfile protects us against concurrent mount/unmount operations.
layer, ok := r.lookup(id)
if !ok {
return nil, nil, ErrLayerUnknown
@ -1400,6 +1565,13 @@ func (r *layerStore) ParentOwners(id string) (uids, gids []int, err error) {
// We don't know which directories to examine.
return nil, nil, ErrLayerNotMounted
}
// Holding r.mountsLockfile protects us against concurrent mount/unmount operations, but we didn’t
// hold it continuously since the time we loaded the mount data; so it’s possible the layer
// was unmounted in the meantime, or mounted elsewhere. Treat that as if we were run after the unmount,
// = a missing mount, not a filesystem error.
if _, err := system.Lstat(layer.MountPoint); errors.Is(err, os.ErrNotExist) {
return nil, nil, ErrLayerNotMounted
}
rootuid, rootgid, err := idtools.GetRootUIDGID(layer.UIDMap, layer.GIDMap)
if err != nil {
return nil, nil, fmt.Errorf("reading root ID values for layer %q: %w", layer.ID, err)
@ -1448,10 +1620,12 @@ func (r *layerStore) ParentOwners(id string) (uids, gids []int, err error) {
return uids, gids, nil
}
// Requires startWriting.
func (r *layerStore) removeName(layer *Layer, name string) {
layer.Names = stringSliceWithoutValue(layer.Names, name)
}
// Requires startWriting.
func (r *layerStore) updateNames(id string, names []string, op updateNameOperation) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to change layer name assignments at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -1486,6 +1660,7 @@ func (r *layerStore) datapath(id, key string) string {
return filepath.Join(r.datadir(id), makeBigDataBaseName(key))
}
// Requires startReading or startWriting.
func (r *layerStore) BigData(id, key string) (io.ReadCloser, error) {
if key == "" {
return nil, fmt.Errorf("can't retrieve layer big data value for empty name: %w", ErrInvalidBigDataName)
@ -1497,6 +1672,7 @@ func (r *layerStore) BigData(id, key string) (io.ReadCloser, error) {
return os.Open(r.datapath(layer.ID, key))
}
// Requires startWriting.
func (r *layerStore) SetBigData(id, key string, data io.Reader) error {
if key == "" {
return fmt.Errorf("can't set empty name for layer big data item: %w", ErrInvalidBigDataName)
@ -1544,6 +1720,7 @@ func (r *layerStore) SetBigData(id, key string, data io.Reader) error {
return nil
}
// Requires startReading or startWriting.
func (r *layerStore) BigDataNames(id string) ([]string, error) {
layer, ok := r.lookup(id)
if !ok {
@ -1552,6 +1729,7 @@ func (r *layerStore) BigDataNames(id string) ([]string, error) {
return copyStringSlice(layer.BigDataNames), nil
}
// Requires startReading or startWriting.
func (r *layerStore) Metadata(id string) (string, error) {
if layer, ok := r.lookup(id); ok {
return layer.Metadata, nil
@ -1559,6 +1737,7 @@ func (r *layerStore) Metadata(id string) (string, error) {
return "", ErrLayerUnknown
}
// Requires startWriting.
func (r *layerStore) SetMetadata(id, metadata string) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to modify layer metadata at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -1575,6 +1754,7 @@ func (r *layerStore) tspath(id string) string {
}
// layerHasIncompleteFlag returns true if layer.Flags contains an incompleteFlag set to true
// The caller must hold r.inProcessLock for reading.
func layerHasIncompleteFlag(layer *Layer) bool {
// layer.Flags[…] is defined to succeed and return ok == false if Flags == nil
if flagValue, ok := layer.Flags[incompleteFlag]; ok {
@ -1585,6 +1765,7 @@ func layerHasIncompleteFlag(layer *Layer) bool {
return false
}
// Requires startWriting.
func (r *layerStore) deleteInternal(id string) error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -1655,6 +1836,7 @@ func (r *layerStore) deleteInternal(id string) error {
return nil
}
// Requires startWriting.
func (r *layerStore) deleteInDigestMap(id string) {
for digest, layers := range r.bycompressedsum {
for i, layerID := range layers {
@ -1676,6 +1858,7 @@ func (r *layerStore) deleteInDigestMap(id string) {
}
}
// Requires startWriting.
func (r *layerStore) Delete(id string) error {
layer, ok := r.lookup(id)
if !ok {
@ -1685,17 +1868,13 @@ func (r *layerStore) Delete(id string) error {
// The layer may already have been explicitly unmounted, but if not, we
// should try to clean that up before we start deleting anything at the
// driver level.
mountCount, err := r.Mounted(id)
if err != nil {
return fmt.Errorf("checking if layer %q is still mounted: %w", id, err)
}
for mountCount > 0 {
if _, err := r.Unmount(id, false); err != nil {
return err
for {
_, err := r.unmount(id, false, true)
if err == ErrLayerNotMounted {
break
}
mountCount, err = r.Mounted(id)
if err != nil {
return fmt.Errorf("checking if layer %q is still mounted: %w", id, err)
return err
}
}
if err := r.deleteInternal(id); err != nil {
@ -1704,11 +1883,13 @@ func (r *layerStore) Delete(id string) error {
return r.saveFor(layer)
}
// Requires startReading or startWriting.
func (r *layerStore) Exists(id string) bool {
_, ok := r.lookup(id)
return ok
}
// Requires startReading or startWriting.
func (r *layerStore) Get(id string) (*Layer, error) {
if layer, ok := r.lookup(id); ok {
return copyLayer(layer), nil
@ -1716,6 +1897,7 @@ func (r *layerStore) Get(id string) (*Layer, error) {
return nil, ErrLayerUnknown
}
// Requires startWriting.
func (r *layerStore) Wipe() error {
if !r.lockfile.IsReadWrite() {
return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -1735,6 +1917,7 @@ func (r *layerStore) Wipe() error {
return nil
}
// Requires startReading or startWriting.
func (r *layerStore) findParentAndLayer(from, to string) (fromID string, toID string, fromLayer, toLayer *Layer, err error) {
var ok bool
toLayer, ok = r.lookup(to)
@ -1759,6 +1942,7 @@ func (r *layerStore) findParentAndLayer(from, to string) (fromID string, toID st
return from, to, fromLayer, toLayer, nil
}
// The caller must hold r.inProcessLock for reading.
func (r *layerStore) layerMappings(layer *Layer) *idtools.IDMappings {
if layer == nil {
return &idtools.IDMappings{}
@ -1766,6 +1950,7 @@ func (r *layerStore) layerMappings(layer *Layer) *idtools.IDMappings {
return idtools.NewIDMappingsFromMaps(layer.UIDMap, layer.GIDMap)
}
// Requires startReading or startWriting.
func (r *layerStore) Changes(from, to string) ([]archive.Change, error) {
from, to, fromLayer, toLayer, err := r.findParentAndLayer(from, to)
if err != nil {
@ -1784,11 +1969,13 @@ func (s *simpleGetCloser) Get(path string) (io.ReadCloser, error) {
return os.Open(filepath.Join(s.path, path))
}
// LOCKING BUG: See the comments in layerStore.Diff
func (s *simpleGetCloser) Close() error {
_, err := s.r.Unmount(s.id, false)
_, err := s.r.unmount(s.id, false, false)
return err
}
// LOCKING BUG: See the comments in layerStore.Diff
func (r *layerStore) newFileGetter(id string) (drivers.FileGetCloser, error) {
if getter, ok := r.driver.(drivers.DiffGetterDriver); ok {
return getter.DiffGetter(id)
@ -1822,6 +2009,7 @@ func writeCompressedDataGoroutine(pwriter *io.PipeWriter, compressor io.WriteClo
err = writeCompressedData(compressor, source)
}
// Requires startReading or startWriting.
func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) {
var metadata storage.Unpacker
@ -1933,6 +2121,8 @@ func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser,
metadata = storage.NewJSONUnpacker(decompressor)
// LOCKING BUG: With btrfs and zfs graph drivers), this uses r.Mount() and r.unmount() holding layerStore only locked for reading
// but they modify in-memory state.
fgetter, err := r.newFileGetter(to)
if err != nil {
errs := multierror.Append(nil, fmt.Errorf("creating file-getter: %w", err))
@ -1968,6 +2158,7 @@ func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser,
return maybeCompressReadCloser(rc)
}
// Requires startReading or startWriting.
func (r *layerStore) DiffSize(from, to string) (size int64, err error) {
var fromLayer, toLayer *Layer
from, to, fromLayer, toLayer, err = r.findParentAndLayer(from, to)
@ -1977,10 +2168,12 @@ func (r *layerStore) DiffSize(from, to string) (size int64, err error) {
return r.driver.DiffSize(to, r.layerMappings(toLayer), from, r.layerMappings(fromLayer), toLayer.MountLabel)
}
// Requires startWriting.
func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error) {
return r.applyDiffWithOptions(to, nil, diff)
}
// Requires startWriting.
func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, diff io.Reader) (size int64, err error) {
if !r.lockfile.IsReadWrite() {
return -1, fmt.Errorf("not allowed to modify layer contents at %q: %w", r.layerdir, ErrStoreIsReadOnly)
@ -2127,6 +2320,7 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions,
return size, err
}
// Requires (startReading or?) startWriting.
func (r *layerStore) DifferTarget(id string) (string, error) {
ddriver, ok := r.driver.(drivers.DriverWithDiffer)
if !ok {
@ -2139,6 +2333,7 @@ func (r *layerStore) DifferTarget(id string) (string, error) {
return ddriver.DifferTarget(layer.ID)
}
// Requires startWriting.
func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error {
ddriver, ok := r.driver.(drivers.DriverWithDiffer)
if !ok {
@ -2177,6 +2372,7 @@ func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string,
return err
}
// Requires startWriting.
func (r *layerStore) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) {
ddriver, ok := r.driver.(drivers.DriverWithDiffer)
if !ok {
@ -2216,6 +2412,7 @@ func (r *layerStore) CleanupStagingDirectory(stagingDirectory string) error {
return ddriver.CleanupStagingDirectory(stagingDirectory)
}
// Requires startReading or startWriting.
func (r *layerStore) layersByDigestMap(m map[digest.Digest][]string, d digest.Digest) ([]Layer, error) {
var layers []Layer
for _, layerID := range m[d] {
@ -2228,10 +2425,12 @@ func (r *layerStore) layersByDigestMap(m map[digest.Digest][]string, d digest.Di
return layers, nil
}
// Requires startReading or startWriting.
func (r *layerStore) LayersByCompressedDigest(d digest.Digest) ([]Layer, error) {
return r.layersByDigestMap(r.bycompressedsum, d)
}
// Requires startReading or startWriting.
func (r *layerStore) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) {
return r.layersByDigestMap(r.byuncompressedsum, d)
}

2
common/vendor/github.com/containers/storage/pkg/lockfile/lockfile.go generated vendored
View File

@ -49,7 +49,7 @@ type Locker interface {
// It might do nothing at all, or it may panic if the caller is not the owner of this lock.
AssertLocked()
// AssertLocked() can be used by callers that _know_ that they hold the lock locked for writing, for sanity checking.
// AssertLockedForWriting() can be used by callers that _know_ that they hold the lock locked for writing, for sanity checking.
// It might do nothing at all, or it may panic if the caller is not the owner of this lock for writing.
AssertLockedForWriting()
}

17
common/vendor/github.com/containers/storage/store.go generated vendored
View File

@ -247,6 +247,8 @@ type Store interface {
// Unmount attempts to unmount an image, given an ID.
// Returns whether or not the layer is still mounted.
// WARNING: The return value may already be obsolete by the time it is available
// to the caller, so it can be used for heuristic sanity checks at best. It should almost always be ignored.
UnmountImage(id string, force bool) (bool, error)
// Mount attempts to mount a layer, image, or container for access, and
@ -265,9 +267,15 @@ type Store interface {
// Unmount attempts to unmount a layer, image, or container, given an ID, a
// name, or a mount path. Returns whether or not the layer is still mounted.
// WARNING: The return value may already be obsolete by the time it is available
// to the caller, so it can be used for heuristic sanity checks at best. It should almost always be ignored.
Unmount(id string, force bool) (bool, error)
// Mounted returns number of times the layer has been mounted.
//
// WARNING: This value might already be obsolete by the time it is returned;
// In situations where concurrent mount/unmount attempts can happen, this field
// should not be used for any decisions, maybe apart from heuristic user warnings.
Mounted(id string) (int, error)
// Changes returns a summary of the changes which would need to be made
@ -2588,7 +2596,7 @@ func (s *store) Unmount(id string, force bool) (bool, error) {
err := s.writeToLayerStore(func(rlstore rwLayerStore) error {
if rlstore.Exists(id) {
var err error
res, err = rlstore.Unmount(id, force)
res, err = rlstore.unmount(id, force, false)
return err
}
return ErrLayerUnknown
@ -3149,8 +3157,11 @@ func (s *store) Shutdown(force bool) ([]string, error) {
}
mounted = append(mounted, layer.ID)
if force {
for layer.MountCount > 0 {
_, err2 := rlstore.Unmount(layer.ID, force)
for {
_, err2 := rlstore.unmount(layer.ID, force, true)
if err2 == ErrLayerNotMounted {
break
}
if err2 != nil {
if err == nil {
err = err2

2
common/vendor/github.com/containers/storage/types/errors.go generated vendored
View File

@ -57,4 +57,6 @@ var (
ErrNotSupported = errors.New("not supported")
// ErrInvalidMappings is returned when the specified mappings are invalid.
ErrInvalidMappings = errors.New("invalid mappings specified")
// ErrNoAvailableIDs is returned when there are not enough unused IDS within the user namespace.
ErrNoAvailableIDs = errors.New("not enough unused IDs in user namespace")
)

4
common/vendor/github.com/containers/storage/userns.go generated vendored
View File

@ -197,7 +197,7 @@ outer:
return 0, err
}
defer func() {
if _, err2 := rlstore.Unmount(clayer.ID, true); err2 != nil {
if _, err2 := rlstore.unmount(clayer.ID, true, false); err2 != nil {
if retErr == nil {
retErr = fmt.Errorf("unmounting temporary layer %#v: %w", clayer.ID, err2)
} else {
@ -264,7 +264,7 @@ func (s *store) getAutoUserNS(options *types.AutoUserNsOptions, image *Image, rl
}
}
if s.autoNsMaxSize > 0 && size > s.autoNsMaxSize {
return nil, nil, fmt.Errorf("the container needs a user namespace with size %q that is bigger than the maximum value allowed with userns=auto %q", size, s.autoNsMaxSize)
return nil, nil, fmt.Errorf("the container needs a user namespace with size %v that is bigger than the maximum value allowed with userns=auto %v", size, s.autoNsMaxSize)
}
}

2
common/vendor/github.com/klauspost/compress/.goreleaser.yml generated vendored
View File

@ -3,7 +3,7 @@
before:
hooks:
- ./gen.sh
- go install mvdan.cc/garble@latest
- go install mvdan.cc/garble@v0.7.2
builds:
-

10
common/vendor/github.com/klauspost/compress/README.md generated vendored
View File

@ -9,7 +9,6 @@ This package provides various compression algorithms.
* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding.
* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently.
* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation.
* [fuzz package](https://github.com/klauspost/compress-fuzz) for fuzz testing all compressors/decompressors here.
[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories)
[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml)
@ -17,6 +16,15 @@ This package provides various compression algorithms.
# changelog
* Dec 11, 2022 (v1.15.13)
* zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691
* zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708
* Oct 26, 2022 (v1.15.12)
* zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680
* gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683
* Sept 26, 2022 (v1.15.11)
* flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678

19
common/vendor/github.com/klauspost/compress/flate/stateless.go generated vendored
View File

@ -86,11 +86,19 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error {
dict = dict[len(dict)-maxStatelessDict:]
}
// For subsequent loops, keep shallow dict reference to avoid alloc+copy.
var inDict []byte
for len(in) > 0 {
todo := in
if len(todo) > maxStatelessBlock-len(dict) {
if len(inDict) > 0 {
if len(todo) > maxStatelessBlock-maxStatelessDict {
todo = todo[:maxStatelessBlock-maxStatelessDict]
}
} else if len(todo) > maxStatelessBlock-len(dict) {
todo = todo[:maxStatelessBlock-len(dict)]
}
inOrg := in
in = in[len(todo):]
uncompressed := todo
if len(dict) > 0 {
@ -102,7 +110,11 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error {
todo = combined
}
// Compress
statelessEnc(&dst, todo, int16(len(dict)))
if len(inDict) == 0 {
statelessEnc(&dst, todo, int16(len(dict)))
} else {
statelessEnc(&dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict)
}
isEof := eof && len(in) == 0
if dst.n == 0 {
@ -119,7 +131,8 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error {
}
if len(in) > 0 {
// Retain a dict if we have more
dict = todo[len(todo)-maxStatelessDict:]
inDict = inOrg[len(uncompressed)-maxStatelessDict:]
dict = nil
dst.Reset()
}
if bw.err != nil {

24
common/vendor/github.com/klauspost/compress/huff0/compress.go generated vendored
View File

@ -365,29 +365,29 @@ func (s *Scratch) countSimple(in []byte) (max int, reuse bool) {
m := uint32(0)
if len(s.prevTable) > 0 {
for i, v := range s.count[:] {
if v == 0 {
continue
}
if v > m {
m = v
}
if v > 0 {
s.symbolLen = uint16(i) + 1
if i >= len(s.prevTable) {
reuse = false
} else {
if s.prevTable[i].nBits == 0 {
reuse = false
}
}
s.symbolLen = uint16(i) + 1
if i >= len(s.prevTable) {
reuse = false
} else if s.prevTable[i].nBits == 0 {
reuse = false
}
}
return int(m), reuse
}
for i, v := range s.count[:] {
if v == 0 {
continue
}
if v > m {
m = v
}
if v > 0 {
s.symbolLen = uint16(i) + 1
}
s.symbolLen = uint16(i) + 1
}
return int(m), false
}

5
common/vendor/github.com/klauspost/compress/zstd/blockdec.go generated vendored
View File

@ -82,8 +82,9 @@ type blockDec struct {
err error
// Check against this crc
checkCRC []byte
// Check against this crc, if hasCRC is true.
checkCRC uint32
hasCRC bool
// Frame to use for singlethreaded decoding.
// Should not be used by the decoder itself since parent may be another frame.

9
common/vendor/github.com/klauspost/compress/zstd/decodeheader.go generated vendored
View File

@ -4,7 +4,6 @@
package zstd
import (
"bytes"
"encoding/binary"
"errors"
"io"
@ -102,8 +101,8 @@ func (h *Header) Decode(in []byte) error {
}
h.HeaderSize += 4
b, in := in[:4], in[4:]
if !bytes.Equal(b, frameMagic) {
if !bytes.Equal(b[1:4], skippableFrameMagic) || b[0]&0xf0 != 0x50 {
if string(b) != frameMagic {
if string(b[1:4]) != skippableFrameMagic || b[0]&0xf0 != 0x50 {
return ErrMagicMismatch
}
if len(in) < 4 {
@ -153,7 +152,7 @@ func (h *Header) Decode(in []byte) error {
}
b, in = in[:size], in[size:]
h.HeaderSize += int(size)
switch size {
switch len(b) {
case 1:
h.DictionaryID = uint32(b[0])
case 2:
@ -183,7 +182,7 @@ func (h *Header) Decode(in []byte) error {
}
b, in = in[:fcsSize], in[fcsSize:]
h.HeaderSize += int(fcsSize)
switch fcsSize {
switch len(b) {
case 1:
h.FrameContentSize = uint64(b[0])
case 2:

39
common/vendor/github.com/klauspost/compress/zstd/decoder.go generated vendored
View File

@ -5,7 +5,6 @@
package zstd
import (
"bytes"
"context"
"encoding/binary"
"io"
@ -459,7 +458,11 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) {
println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp)
}
if !d.o.ignoreChecksum && len(next.b) > 0 {
if d.o.ignoreChecksum {
return true
}
if len(next.b) > 0 {
n, err := d.current.crc.Write(next.b)
if err == nil {
if n != len(next.b) {
@ -467,18 +470,16 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) {
}
}
}
if next.err == nil && next.d != nil && len(next.d.checkCRC) != 0 {
got := d.current.crc.Sum64()
var tmp [4]byte
binary.LittleEndian.PutUint32(tmp[:], uint32(got))
if !d.o.ignoreChecksum && !bytes.Equal(tmp[:], next.d.checkCRC) {
if next.err == nil && next.d != nil && next.d.hasCRC {
got := uint32(d.current.crc.Sum64())
if got != next.d.checkCRC {
if debugDecoder {
println("CRC Check Failed:", tmp[:], " (got) !=", next.d.checkCRC, "(on stream)")
printf("CRC Check Failed: %08x (got) != %08x (on stream)\n", got, next.d.checkCRC)
}
d.current.err = ErrCRCMismatch
} else {
if debugDecoder {
println("CRC ok", tmp[:])
printf("CRC ok %08x\n", got)
}
}
}
@ -918,18 +919,22 @@ decodeStream:
println("next block returned error:", err)
}
dec.err = err
dec.checkCRC = nil
dec.hasCRC = false
if dec.Last && frame.HasCheckSum && err == nil {
crc, err := frame.rawInput.readSmall(4)
if err != nil {
if len(crc) < 4 {
if err == nil {
err = io.ErrUnexpectedEOF
}
println("CRC missing?", err)
dec.err = err
}
var tmp [4]byte
copy(tmp[:], crc)
dec.checkCRC = tmp[:]
if debugDecoder {
println("found crc to check:", dec.checkCRC)
} else {
dec.checkCRC = binary.LittleEndian.Uint32(crc)
dec.hasCRC = true
if debugDecoder {
printf("found crc to check: %08x\n", dec.checkCRC)
}
}
}
err = dec.err

5
common/vendor/github.com/klauspost/compress/zstd/dict.go generated vendored
View File

@ -1,7 +1,6 @@
package zstd
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
@ -20,7 +19,7 @@ type dict struct {
content []byte
}
var dictMagic = [4]byte{0x37, 0xa4, 0x30, 0xec}
const dictMagic = "\x37\xa4\x30\xec"
// ID returns the dictionary id or 0 if d is nil.
func (d *dict) ID() uint32 {
@ -50,7 +49,7 @@ func loadDict(b []byte) (*dict, error) {
ofDec: sequenceDec{fse: &fseDecoder{}},
mlDec: sequenceDec{fse: &fseDecoder{}},
}
if !bytes.Equal(b[:4], dictMagic[:]) {
if string(b[:4]) != dictMagic {
return nil, ErrMagicMismatch
}
d.id = binary.LittleEndian.Uint32(b[4:8])

26
common/vendor/github.com/klauspost/compress/zstd/enc_base.go generated vendored
View File

@ -16,6 +16,7 @@ type fastBase struct {
cur int32
// maximum offset. Should be at least 2x block size.
maxMatchOff int32
bufferReset int32
hist []byte
crc *xxhash.Digest
tmp [8]byte
@ -56,8 +57,8 @@ func (e *fastBase) Block() *blockEnc {
}
func (e *fastBase) addBlock(src []byte) int32 {
if debugAsserts && e.cur > bufferReset {
panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, bufferReset))
if debugAsserts && e.cur > e.bufferReset {
panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, e.bufferReset))
}
// check if we have space already
if len(e.hist)+len(src) > cap(e.hist) {
@ -126,24 +127,7 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 {
panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize))
}
}
a := src[s:]
b := src[t:]
b = b[:len(a)]
end := int32((len(a) >> 3) << 3)
for i := int32(0); i < end; i += 8 {
if diff := load6432(a, i) ^ load6432(b, i); diff != 0 {
return i + int32(bits.TrailingZeros64(diff)>>3)
}
}
a = a[end:]
b = b[end:]
for i := range a {
if a[i] != b[i] {
return int32(i) + end
}
}
return int32(len(a)) + end
return int32(matchLen(src[s:], src[t:]))
}
// Reset the encoding table.
@ -171,7 +155,7 @@ func (e *fastBase) resetBase(d *dict, singleBlock bool) {
// We offset current position so everything will be out of reach.
// If above reset line, history will be purged.
if e.cur < bufferReset {
if e.cur < e.bufferReset {
e.cur += e.maxMatchOff + int32(len(e.hist))
}
e.hist = e.hist[:0]

63
common/vendor/github.com/klauspost/compress/zstd/enc_best.go generated vendored
View File

@ -85,14 +85,10 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) {
)
// Protect against e.cur wraparound.
for e.cur >= bufferReset {
for e.cur >= e.bufferReset-int32(len(e.hist)) {
if len(e.hist) == 0 {
for i := range e.table[:] {
e.table[i] = prevEntry{}
}
for i := range e.longTable[:] {
e.longTable[i] = prevEntry{}
}
e.table = [bestShortTableSize]prevEntry{}
e.longTable = [bestLongTableSize]prevEntry{}
e.cur = e.maxMatchOff
break
}
@ -193,8 +189,8 @@ encodeLoop:
panic("offset0 was 0")
}
bestOf := func(a, b match) match {
if a.est+(a.s-b.s)*bitsPerByte>>10 < b.est+(b.s-a.s)*bitsPerByte>>10 {
bestOf := func(a, b *match) *match {
if a.est-b.est+(a.s-b.s)*bitsPerByte>>10 < 0 {
return a
}
return b
@ -220,22 +216,26 @@ encodeLoop:
return m
}
best := bestOf(matchAt(candidateL.offset-e.cur, s, uint32(cv), -1), matchAt(candidateL.prev-e.cur, s, uint32(cv), -1))
best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1))
best = bestOf(best, matchAt(candidateS.prev-e.cur, s, uint32(cv), -1))
m1 := matchAt(candidateL.offset-e.cur, s, uint32(cv), -1)
m2 := matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)
m3 := matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)
m4 := matchAt(candidateS.prev-e.cur, s, uint32(cv), -1)
best := bestOf(bestOf(&m1, &m2), bestOf(&m3, &m4))
if canRepeat && best.length < goodEnough {
cv32 := uint32(cv >> 8)
spp := s + 1
best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1))
best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2))
best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3))
m1 := matchAt(spp-offset1, spp, cv32, 1)
m2 := matchAt(spp-offset2, spp, cv32, 2)
m3 := matchAt(spp-offset3, spp, cv32, 3)
best = bestOf(bestOf(best, &m1), bestOf(&m2, &m3))
if best.length > 0 {
cv32 = uint32(cv >> 24)
spp += 2
best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1))
best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2))
best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3))
m1 := matchAt(spp-offset1, spp, cv32, 1)
m2 := matchAt(spp-offset2, spp, cv32, 2)
m3 := matchAt(spp-offset3, spp, cv32, 3)
best = bestOf(bestOf(best, &m1), bestOf(&m2, &m3))
}
}
// Load next and check...
@ -262,26 +262,33 @@ encodeLoop:
candidateL2 := e.longTable[hashLen(cv2, bestLongTableBits, bestLongLen)]
// Short at s+1
best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1))
m1 := matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)
// Long at s+1, s+2
best = bestOf(best, matchAt(candidateL.offset-e.cur, s, uint32(cv), -1))
best = bestOf(best, matchAt(candidateL.prev-e.cur, s, uint32(cv), -1))
best = bestOf(best, matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1))
best = bestOf(best, matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1))
m2 := matchAt(candidateL.offset-e.cur, s, uint32(cv), -1)
m3 := matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)
m4 := matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1)
m5 := matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1)
best = bestOf(bestOf(bestOf(best, &m1), &m2), bestOf(bestOf(&m3, &m4), &m5))
if false {
// Short at s+3.
// Too often worse...
best = bestOf(best, matchAt(e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1))
m := matchAt(e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1)
best = bestOf(best, &m)
}
// See if we can find a better match by checking where the current best ends.
// Use that offset to see if we can find a better full match.
if sAt := best.s + best.length; sAt < sLimit {
nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen)
candidateEnd := e.longTable[nextHashL]
if pos := candidateEnd.offset - e.cur - best.length; pos >= 0 {
bestEnd := bestOf(best, matchAt(pos, best.s, load3232(src, best.s), -1))
if pos := candidateEnd.prev - e.cur - best.length; pos >= 0 {
bestEnd = bestOf(bestEnd, matchAt(pos, best.s, load3232(src, best.s), -1))
// Start check at a fixed offset to allow for a few mismatches.
// For this compression level 2 yields the best results.
const skipBeginning = 2
if pos := candidateEnd.offset - e.cur - best.length + skipBeginning; pos >= 0 {
m := matchAt(pos, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1)
bestEnd := bestOf(best, &m)
if pos := candidateEnd.prev - e.cur - best.length + skipBeginning; pos >= 0 {
m := matchAt(pos, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1)
bestEnd = bestOf(bestEnd, &m)
}
best = bestEnd
}

12
common/vendor/github.com/klauspost/compress/zstd/enc_better.go generated vendored
View File

@ -62,14 +62,10 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) {
)
// Protect against e.cur wraparound.
for e.cur >= bufferReset {
for e.cur >= e.bufferReset-int32(len(e.hist)) {
if len(e.hist) == 0 {
for i := range e.table[:] {
e.table[i] = tableEntry{}
}
for i := range e.longTable[:] {
e.longTable[i] = prevEntry{}
}
e.table = [betterShortTableSize]tableEntry{}
e.longTable = [betterLongTableSize]prevEntry{}
e.cur = e.maxMatchOff
break
}
@ -587,7 +583,7 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) {
)
// Protect against e.cur wraparound.
for e.cur >= bufferReset {
for e.cur >= e.bufferReset-int32(len(e.hist)) {
if len(e.hist) == 0 {
for i := range e.table[:] {
e.table[i] = tableEntry{}

16
common/vendor/github.com/klauspost/compress/zstd/enc_dfast.go generated vendored
View File

@ -44,14 +44,10 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) {
)
// Protect against e.cur wraparound.
for e.cur >= bufferReset {
for e.cur >= e.bufferReset-int32(len(e.hist)) {
if len(e.hist) == 0 {
for i := range e.table[:] {
e.table[i] = tableEntry{}
}
for i := range e.longTable[:] {
e.longTable[i] = tableEntry{}
}
e.table = [dFastShortTableSize]tableEntry{}
e.longTable = [dFastLongTableSize]tableEntry{}
e.cur = e.maxMatchOff
break
}
@ -388,7 +384,7 @@ func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) {
)
// Protect against e.cur wraparound.
if e.cur >= bufferReset {
if e.cur >= e.bufferReset {
for i := range e.table[:] {
e.table[i] = tableEntry{}
}
@ -685,7 +681,7 @@ encodeLoop:
}
// We do not store history, so we must offset e.cur to avoid false matches for next user.
if e.cur < bufferReset {
if e.cur < e.bufferReset {
e.cur += int32(len(src))
}
}
@ -700,7 +696,7 @@ func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) {
)
// Protect against e.cur wraparound.
for e.cur >= bufferReset {
for e.cur >= e.bufferReset-int32(len(e.hist)) {
if len(e.hist) == 0 {
for i := range e.table[:] {
e.table[i] = tableEntry{}

12
common/vendor/github.com/klauspost/compress/zstd/enc_fast.go generated vendored
View File

@ -43,7 +43,7 @@ func (e *fastEncoder) Encode(blk *blockEnc, src []byte) {
)
// Protect against e.cur wraparound.
for e.cur >= bufferReset {
for e.cur >= e.bufferReset-int32(len(e.hist)) {
if len(e.hist) == 0 {
for i := range e.table[:] {
e.table[i] = tableEntry{}
@ -310,7 +310,7 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) {
}
// Protect against e.cur wraparound.
if e.cur >= bufferReset {
if e.cur >= e.bufferReset {
for i := range e.table[:] {
e.table[i] = tableEntry{}
}
@ -538,7 +538,7 @@ encodeLoop:
println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits)
}
// We do not store history, so we must offset e.cur to avoid false matches for next user.
if e.cur < bufferReset {
if e.cur < e.bufferReset {
e.cur += int32(len(src))
}
}
@ -555,11 +555,9 @@ func (e *fastEncoderDict) Encode(blk *blockEnc, src []byte) {
return
}
// Protect against e.cur wraparound.
for e.cur >= bufferReset {
for e.cur >= e.bufferReset-int32(len(e.hist)) {
if len(e.hist) == 0 {
for i := range e.table[:] {
e.table[i] = tableEntry{}
}
e.table = [tableSize]tableEntry{}
e.cur = e.maxMatchOff
break
}

35
common/vendor/github.com/klauspost/compress/zstd/encoder.go generated vendored
View File

@ -8,6 +8,7 @@ import (
"crypto/rand"
"fmt"
"io"
"math"
rdebug "runtime/debug"
"sync"
@ -639,3 +640,37 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
}
return dst
}
// MaxEncodedSize returns the expected maximum
// size of an encoded block or stream.
func (e *Encoder) MaxEncodedSize(size int) int {
frameHeader := 4 + 2 // magic + frame header & window descriptor
if e.o.dict != nil {
frameHeader += 4
}
// Frame content size:
if size < 256 {
frameHeader++
} else if size < 65536+256 {
frameHeader += 2
} else if size < math.MaxInt32 {
frameHeader += 4
} else {
frameHeader += 8
}
// Final crc
if e.o.crc {
frameHeader += 4
}
// Max overhead is 3 bytes/block.
// There cannot be 0 blocks.
blocks := (size + e.o.blockSize) / e.o.blockSize
// Combine, add padding.
maxSz := frameHeader + 3*blocks + size
if e.o.pad > 1 {
maxSz += calcSkippableFrame(int64(maxSz), int64(e.o.pad))
}
return maxSz
}

15
common/vendor/github.com/klauspost/compress/zstd/encoder_options.go generated vendored
View File

@ -3,6 +3,7 @@ package zstd
import (
"errors"
"fmt"
"math"
"runtime"
"strings"
)
@ -47,22 +48,22 @@ func (o encoderOptions) encoder() encoder {
switch o.level {
case SpeedFastest:
if o.dict != nil {
return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}
return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}
}
return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}
return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}
case SpeedDefault:
if o.dict != nil {
return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}}
return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}}
}
return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}
return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}
case SpeedBetterCompression:
if o.dict != nil {
return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}
return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}
}
return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}
return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}
case SpeedBestCompression:
return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}
return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}
}
panic("unknown compression level")
}

37
common/vendor/github.com/klauspost/compress/zstd/framedec.go generated vendored
View File

@ -5,7 +5,7 @@
package zstd
import (
"bytes"
"encoding/binary"
"encoding/hex"
"errors"
"io"
@ -43,9 +43,9 @@ const (
MaxWindowSize = 1 << 29
)
var (
frameMagic = []byte{0x28, 0xb5, 0x2f, 0xfd}
skippableFrameMagic = []byte{0x2a, 0x4d, 0x18}
const (
frameMagic = "\x28\xb5\x2f\xfd"
skippableFrameMagic = "\x2a\x4d\x18"
)
func newFrameDec(o decoderOptions) *frameDec {
@ -89,9 +89,9 @@ func (d *frameDec) reset(br byteBuffer) error {
copy(signature[1:], b)
}
if !bytes.Equal(signature[1:4], skippableFrameMagic) || signature[0]&0xf0 != 0x50 {
if string(signature[1:4]) != skippableFrameMagic || signature[0]&0xf0 != 0x50 {
if debugDecoder {
println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString(skippableFrameMagic))
println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString([]byte(skippableFrameMagic)))
}
// Break if not skippable frame.
break
@ -114,9 +114,9 @@ func (d *frameDec) reset(br byteBuffer) error {
return err
}
}
if !bytes.Equal(signature[:], frameMagic) {
if string(signature[:]) != frameMagic {
if debugDecoder {
println("Got magic numbers: ", signature, "want:", frameMagic)
println("Got magic numbers: ", signature, "want:", []byte(frameMagic))
}
return ErrMagicMismatch
}
@ -167,7 +167,7 @@ func (d *frameDec) reset(br byteBuffer) error {
return err
}
var id uint32
switch size {
switch len(b) {
case 1:
id = uint32(b[0])
case 2:
@ -204,7 +204,7 @@ func (d *frameDec) reset(br byteBuffer) error {
println("Reading Frame content", err)
return err
}
switch fcsSize {
switch len(b) {
case 1:
d.FrameContentSize = uint64(b[0])
case 2:
@ -305,7 +305,7 @@ func (d *frameDec) checkCRC() error {
}
// We can overwrite upper tmp now
want, err := d.rawInput.readSmall(4)
buf, err := d.rawInput.readSmall(4)
if err != nil {
println("CRC missing?", err)
return err
@ -315,22 +315,17 @@ func (d *frameDec) checkCRC() error {
return nil
}
var tmp [4]byte
got := d.crc.Sum64()
// Flip to match file order.
tmp[0] = byte(got >> 0)
tmp[1] = byte(got >> 8)
tmp[2] = byte(got >> 16)
tmp[3] = byte(got >> 24)
want := binary.LittleEndian.Uint32(buf[:4])
got := uint32(d.crc.Sum64())
if !bytes.Equal(tmp[:], want) {
if got != want {
if debugDecoder {
println("CRC Check Failed:", tmp[:], "!=", want)
printf("CRC check failed: got %08x, want %08x\n", got, want)
}
return ErrCRCMismatch
}
if debugDecoder {
println("CRC ok", tmp[:])
printf("CRC ok %08x\n", got)
}
return nil
}

49
common/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md generated vendored
View File

@ -2,12 +2,7 @@
VENDORED: Go to [github.com/cespare/xxhash](https://github.com/cespare/xxhash) for original package.
[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash)
[![Build Status](https://travis-ci.org/cespare/xxhash.svg?branch=master)](https://travis-ci.org/cespare/xxhash)
xxhash is a Go implementation of the 64-bit
[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a
high-quality hashing algorithm that is much faster than anything in the Go
standard library.
@ -28,31 +23,49 @@ func (*Digest) WriteString(string) (int, error)
func (*Digest) Sum64() uint64
```
This implementation provides a fast pure-Go implementation and an even faster
assembly implementation for amd64.
The package is written with optimized pure Go and also contains even faster
assembly implementations for amd64 and arm64. If desired, the `purego` build tag
opts into using the Go code even on those architectures.
[xxHash]: http://cyan4973.github.io/xxHash/
## Compatibility
This package is in a module and the latest code is in version 2 of the module.
You need a version of Go with at least "minimal module compatibility" to use
github.com/cespare/xxhash/v2:
* 1.9.7+ for Go 1.9
* 1.10.3+ for Go 1.10
* Go 1.11 or later
I recommend using the latest release of Go.
## Benchmarks
Here are some quick benchmarks comparing the pure-Go and assembly
implementations of Sum64.
| input size | purego | asm |
| --- | --- | --- |
| 5 B | 979.66 MB/s | 1291.17 MB/s |
| 100 B | 7475.26 MB/s | 7973.40 MB/s |
| 4 KB | 17573.46 MB/s | 17602.65 MB/s |
| 10 MB | 17131.46 MB/s | 17142.16 MB/s |
| input size | purego | asm |
| ---------- | --------- | --------- |
| 4 B | 1.3 GB/s | 1.2 GB/s |
| 16 B | 2.9 GB/s | 3.5 GB/s |
| 100 B | 6.9 GB/s | 8.1 GB/s |
| 4 KB | 11.7 GB/s | 16.7 GB/s |
| 10 MB | 12.0 GB/s | 17.3 GB/s |
These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using
the following commands under Go 1.11.2:
These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C
CPU using the following commands under Go 1.19.2:
```
$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes'
$ go test -benchtime 10s -bench '/xxhash,direct,bytes'
benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$')
benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$')
```
## Projects using this package
- [InfluxDB](https://github.com/influxdata/influxdb)
- [Prometheus](https://github.com/prometheus/prometheus)
- [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
- [FreeCache](https://github.com/coocood/freecache)
- [FastCache](https://github.com/VictoriaMetrics/fastcache)

47
common/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go generated vendored
View File

@ -18,19 +18,11 @@ const (
prime5 uint64 = 2870177450012600261
)
// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
// possible in the Go code is worth a small (but measurable) performance boost
// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
// convenience in the Go code in a few places where we need to intentionally
// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
// result overflows a uint64).
var (
prime1v = prime1
prime2v = prime2
prime3v = prime3
prime4v = prime4
prime5v = prime5
)
// Store the primes in an array as well.
//
// The consts are used when possible in Go code to avoid MOVs but we need a
// contiguous array of the assembly code.
var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5}
// Digest implements hash.Hash64.
type Digest struct {
@ -52,10 +44,10 @@ func New() *Digest {
// Reset clears the Digest's state so that it can be reused.
func (d *Digest) Reset() {
d.v1 = prime1v + prime2
d.v1 = primes[0] + prime2
d.v2 = prime2
d.v3 = 0
d.v4 = -prime1v
d.v4 = -primes[0]
d.total = 0
d.n = 0
}
@ -71,21 +63,23 @@ func (d *Digest) Write(b []byte) (n int, err error) {
n = len(b)
d.total += uint64(n)
memleft := d.mem[d.n&(len(d.mem)-1):]
if d.n+n < 32 {
// This new data doesn't even fill the current block.
copy(d.mem[d.n:], b)
copy(memleft, b)
d.n += n
return
}
if d.n > 0 {
// Finish off the partial block.
copy(d.mem[d.n:], b)
c := copy(memleft, b)
d.v1 = round(d.v1, u64(d.mem[0:8]))
d.v2 = round(d.v2, u64(d.mem[8:16]))
d.v3 = round(d.v3, u64(d.mem[16:24]))
d.v4 = round(d.v4, u64(d.mem[24:32]))
b = b[32-d.n:]
b = b[c:]
d.n = 0
}
@ -135,21 +129,20 @@ func (d *Digest) Sum64() uint64 {
h += d.total
i, end := 0, d.n
for ; i+8 <= end; i += 8 {
k1 := round(0, u64(d.mem[i:i+8]))
b := d.mem[:d.n&(len(d.mem)-1)]
for ; len(b) >= 8; b = b[8:] {
k1 := round(0, u64(b[:8]))
h ^= k1
h = rol27(h)*prime1 + prime4
}
if i+4 <= end {
h ^= uint64(u32(d.mem[i:i+4])) * prime1
if len(b) >= 4 {
h ^= uint64(u32(b[:4])) * prime1
h = rol23(h)*prime2 + prime3
i += 4
b = b[4:]
}
for i < end {
h ^= uint64(d.mem[i]) * prime5
for ; len(b) > 0; b = b[1:] {
h ^= uint64(b[0]) * prime5
h = rol11(h) * prime1
i++
}
h ^= h >> 33

308
common/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s generated vendored
View File

@ -1,3 +1,4 @@
//go:build !appengine && gc && !purego && !noasm
// +build !appengine
// +build gc
// +build !purego
@ -5,212 +6,205 @@
#include "textflag.h"
// Register allocation:
// AX h
// SI pointer to advance through b
// DX n
// BX loop end
// R8 v1, k1
// R9 v2
// R10 v3
// R11 v4
// R12 tmp
// R13 prime1v
// R14 prime2v
// DI prime4v
// Registers:
#define h AX
#define d AX
#define p SI // pointer to advance through b
#define n DX
#define end BX // loop end
#define v1 R8
#define v2 R9
#define v3 R10
#define v4 R11
#define x R12
#define prime1 R13
#define prime2 R14
#define prime4 DI
// round reads from and advances the buffer pointer in SI.
// It assumes that R13 has prime1v and R14 has prime2v.
#define round(r) \
MOVQ (SI), R12 \
ADDQ $8, SI \
IMULQ R14, R12 \
ADDQ R12, r \
ROLQ $31, r \
IMULQ R13, r
#define round(acc, x) \
IMULQ prime2, x \
ADDQ x, acc \
ROLQ $31, acc \
IMULQ prime1, acc
// mergeRound applies a merge round on the two registers acc and val.
// It assumes that R13 has prime1v, R14 has prime2v, and DI has prime4v.
#define mergeRound(acc, val) \
IMULQ R14, val \
ROLQ $31, val \
IMULQ R13, val \
XORQ val, acc \
IMULQ R13, acc \
ADDQ DI, acc
// round0 performs the operation x = round(0, x).
#define round0(x) \
IMULQ prime2, x \
ROLQ $31, x \
IMULQ prime1, x
// mergeRound applies a merge round on the two registers acc and x.
// It assumes that prime1, prime2, and prime4 have been loaded.
#define mergeRound(acc, x) \
round0(x) \
XORQ x, acc \
IMULQ prime1, acc \
ADDQ prime4, acc
// blockLoop processes as many 32-byte blocks as possible,
// updating v1, v2, v3, and v4. It assumes that there is at least one block
// to process.
#define blockLoop() \
loop: \
MOVQ +0(p), x \
round(v1, x) \
MOVQ +8(p), x \
round(v2, x) \
MOVQ +16(p), x \
round(v3, x) \
MOVQ +24(p), x \
round(v4, x) \
ADDQ $32, p \
CMPQ p, end \
JLE loop
// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOSPLIT, $0-32
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
// Load fixed primes.
MOVQ ·prime1v(SB), R13
MOVQ ·prime2v(SB), R14
MOVQ ·prime4v(SB), DI
MOVQ ·primes+0(SB), prime1
MOVQ ·primes+8(SB), prime2
MOVQ ·primes+24(SB), prime4
// Load slice.
MOVQ b_base+0(FP), SI
MOVQ b_len+8(FP), DX
LEAQ (SI)(DX*1), BX
MOVQ b_base+0(FP), p
MOVQ b_len+8(FP), n
LEAQ (p)(n*1), end
// The first loop limit will be len(b)-32.
SUBQ $32, BX
SUBQ $32, end
// Check whether we have at least one block.
CMPQ DX, $32
CMPQ n, $32
JLT noBlocks
// Set up initial state (v1, v2, v3, v4).
MOVQ R13, R8
ADDQ R14, R8
MOVQ R14, R9
XORQ R10, R10
XORQ R11, R11
SUBQ R13, R11
MOVQ prime1, v1
ADDQ prime2, v1
MOVQ prime2, v2
XORQ v3, v3
XORQ v4, v4
SUBQ prime1, v4
// Loop until SI > BX.
blockLoop:
round(R8)
round(R9)
round(R10)
round(R11)
blockLoop()
CMPQ SI, BX
JLE blockLoop
MOVQ v1, h
ROLQ $1, h
MOVQ v2, x
ROLQ $7, x
ADDQ x, h
MOVQ v3, x
ROLQ $12, x
ADDQ x, h
MOVQ v4, x
ROLQ $18, x
ADDQ x, h
MOVQ R8, AX
ROLQ $1, AX
MOVQ R9, R12
ROLQ $7, R12
ADDQ R12, AX
MOVQ R10, R12
ROLQ $12, R12
ADDQ R12, AX
MOVQ R11, R12
ROLQ $18, R12
ADDQ R12, AX
mergeRound(AX, R8)
mergeRound(AX, R9)
mergeRound(AX, R10)
mergeRound(AX, R11)
mergeRound(h, v1)
mergeRound(h, v2)
mergeRound(h, v3)
mergeRound(h, v4)
JMP afterBlocks
noBlocks:
MOVQ ·prime5v(SB), AX
MOVQ ·primes+32(SB), h
afterBlocks:
ADDQ DX, AX
ADDQ n, h
// Right now BX has len(b)-32, and we want to loop until SI > len(b)-8.
ADDQ $24, BX
ADDQ $24, end
CMPQ p, end
JG try4
CMPQ SI, BX
JG fourByte
loop8:
MOVQ (p), x
ADDQ $8, p
round0(x)
XORQ x, h
ROLQ $27, h
IMULQ prime1, h
ADDQ prime4, h
wordLoop:
// Calculate k1.
MOVQ (SI), R8
ADDQ $8, SI
IMULQ R14, R8
ROLQ $31, R8
IMULQ R13, R8
CMPQ p, end
JLE loop8
XORQ R8, AX
ROLQ $27, AX
IMULQ R13, AX
ADDQ DI, AX
try4:
ADDQ $4, end
CMPQ p, end
JG try1
CMPQ SI, BX
JLE wordLoop
MOVL (p), x
ADDQ $4, p
IMULQ prime1, x
XORQ x, h
fourByte:
ADDQ $4, BX
CMPQ SI, BX
JG singles
ROLQ $23, h
IMULQ prime2, h
ADDQ ·primes+16(SB), h
MOVL (SI), R8
ADDQ $4, SI
IMULQ R13, R8
XORQ R8, AX
ROLQ $23, AX
IMULQ R14, AX
ADDQ ·prime3v(SB), AX
singles:
ADDQ $4, BX
CMPQ SI, BX
try1:
ADDQ $4, end
CMPQ p, end
JGE finalize
singlesLoop:
MOVBQZX (SI), R12
ADDQ $1, SI
IMULQ ·prime5v(SB), R12
XORQ R12, AX
loop1:
MOVBQZX (p), x
ADDQ $1, p
IMULQ ·primes+32(SB), x
XORQ x, h
ROLQ $11, h
IMULQ prime1, h
ROLQ $11, AX
IMULQ R13, AX
CMPQ SI, BX
JL singlesLoop
CMPQ p, end
JL loop1
finalize:
MOVQ AX, R12
SHRQ $33, R12
XORQ R12, AX
IMULQ R14, AX
MOVQ AX, R12
SHRQ $29, R12
XORQ R12, AX
IMULQ ·prime3v(SB), AX
MOVQ AX, R12
SHRQ $32, R12
XORQ R12, AX
MOVQ h, x
SHRQ $33, x
XORQ x, h
IMULQ prime2, h
MOVQ h, x
SHRQ $29, x
XORQ x, h
IMULQ ·primes+16(SB), h
MOVQ h, x
SHRQ $32, x
XORQ x, h
MOVQ AX, ret+24(FP)
MOVQ h, ret+24(FP)
RET
// writeBlocks uses the same registers as above except that it uses AX to store
// the d pointer.
// func writeBlocks(d *Digest, b []byte) int
TEXT ·writeBlocks(SB), NOSPLIT, $0-40
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
// Load fixed primes needed for round.
MOVQ ·prime1v(SB), R13
MOVQ ·prime2v(SB), R14
MOVQ ·primes+0(SB), prime1
MOVQ ·primes+8(SB), prime2
// Load slice.
MOVQ b_base+8(FP), SI
MOVQ b_len+16(FP), DX
LEAQ (SI)(DX*1), BX
SUBQ $32, BX
MOVQ b_base+8(FP), p
MOVQ b_len+16(FP), n
LEAQ (p)(n*1), end
SUBQ $32, end
// Load vN from d.
MOVQ d+0(FP), AX
MOVQ 0(AX), R8 // v1
MOVQ 8(AX), R9 // v2
MOVQ 16(AX), R10 // v3
MOVQ 24(AX), R11 // v4
MOVQ s+0(FP), d
MOVQ 0(d), v1
MOVQ 8(d), v2
MOVQ 16(d), v3
MOVQ 24(d), v4
// We don't need to check the loop condition here; this function is
// always called with at least one block of data to process.
blockLoop:
round(R8)
round(R9)
round(R10)
round(R11)
CMPQ SI, BX
JLE blockLoop
blockLoop()
// Copy vN back to d.
MOVQ R8, 0(AX)
MOVQ R9, 8(AX)
MOVQ R10, 16(AX)
MOVQ R11, 24(AX)
MOVQ v1, 0(d)
MOVQ v2, 8(d)
MOVQ v3, 16(d)
MOVQ v4, 24(d)
// The number of bytes written is SI minus the old base pointer.
SUBQ b_base+8(FP), SI
MOVQ SI, ret+32(FP)
// The number of bytes written is p minus the old base pointer.
SUBQ b_base+8(FP), p
MOVQ p, ret+32(FP)
RET

136
common/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s generated vendored
View File

@ -1,13 +1,17 @@
// +build gc,!purego,!noasm
//go:build !appengine && gc && !purego && !noasm
// +build !appengine
// +build gc
// +build !purego
// +build !noasm
#include "textflag.h"
// Register allocation.
// Registers:
#define digest R1
#define h R2 // Return value.
#define p R3 // Input pointer.
#define len R4
#define nblocks R5 // len / 32.
#define h R2 // return value
#define p R3 // input pointer
#define n R4 // input length
#define nblocks R5 // n / 32
#define prime1 R7
#define prime2 R8
#define prime3 R9
@ -25,60 +29,52 @@
#define round(acc, x) \
MADD prime2, acc, x, acc \
ROR $64-31, acc \
MUL prime1, acc \
MUL prime1, acc
// x = round(0, x).
// round0 performs the operation x = round(0, x).
#define round0(x) \
MUL prime2, x \
ROR $64-31, x \
MUL prime1, x \
MUL prime1, x
#define mergeRound(x) \
round0(x) \
EOR x, h \
MADD h, prime4, prime1, h \
#define mergeRound(acc, x) \
round0(x) \
EOR x, acc \
MADD acc, prime4, prime1, acc
// Update v[1-4] with 32-byte blocks. Assumes len >= 32.
#define blocksLoop() \
LSR $5, len, nblocks \
PCALIGN $16 \
loop: \
LDP.P 32(p), (x1, x2) \
round(v1, x1) \
LDP -16(p), (x3, x4) \
round(v2, x2) \
SUB $1, nblocks \
round(v3, x3) \
round(v4, x4) \
CBNZ nblocks, loop \
// The primes are repeated here to ensure that they're stored
// in a contiguous array, so we can load them with LDP.
DATA primes<> +0(SB)/8, $11400714785074694791
DATA primes<> +8(SB)/8, $14029467366897019727
DATA primes<>+16(SB)/8, $1609587929392839161
DATA primes<>+24(SB)/8, $9650029242287828579
DATA primes<>+32(SB)/8, $2870177450012600261
GLOBL primes<>(SB), NOPTR+RODATA, $40
// blockLoop processes as many 32-byte blocks as possible,
// updating v1, v2, v3, and v4. It assumes that n >= 32.
#define blockLoop() \
LSR $5, n, nblocks \
PCALIGN $16 \
loop: \
LDP.P 16(p), (x1, x2) \
LDP.P 16(p), (x3, x4) \
round(v1, x1) \
round(v2, x2) \
round(v3, x3) \
round(v4, x4) \
SUB $1, nblocks \
CBNZ nblocks, loop
// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOFRAME+NOSPLIT, $0-32
LDP b_base+0(FP), (p, len)
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
LDP b_base+0(FP), (p, n)
LDP primes<> +0(SB), (prime1, prime2)
LDP primes<>+16(SB), (prime3, prime4)
MOVD primes<>+32(SB), prime5
LDP ·primes+0(SB), (prime1, prime2)
LDP ·primes+16(SB), (prime3, prime4)
MOVD ·primes+32(SB), prime5
CMP $32, len
CSEL LO, prime5, ZR, h // if len < 32 { h = prime5 } else { h = 0 }
BLO afterLoop
CMP $32, n
CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 }
BLT afterLoop
ADD prime1, prime2, v1
MOVD prime2, v2
MOVD $0, v3
NEG prime1, v4
blocksLoop()
blockLoop()
ROR $64-1, v1, x1
ROR $64-7, v2, x2
@ -88,71 +84,75 @@ TEXT ·Sum64(SB), NOFRAME+NOSPLIT, $0-32
ADD x3, x4
ADD x2, x4, h
mergeRound(v1)
mergeRound(v2)
mergeRound(v3)
mergeRound(v4)
mergeRound(h, v1)
mergeRound(h, v2)
mergeRound(h, v3)
mergeRound(h, v4)
afterLoop:
ADD len, h
ADD n, h
TBZ $4, len, try8
TBZ $4, n, try8
LDP.P 16(p), (x1, x2)
round0(x1)
// NOTE: here and below, sequencing the EOR after the ROR (using a
// rotated register) is worth a small but measurable speedup for small
// inputs.
ROR $64-27, h
EOR x1 @> 64-27, h, h
MADD h, prime4, prime1, h
round0(x2)
ROR $64-27, h
EOR x2 @> 64-27, h
EOR x2 @> 64-27, h, h
MADD h, prime4, prime1, h
try8:
TBZ $3, len, try4
TBZ $3, n, try4
MOVD.P 8(p), x1
round0(x1)
ROR $64-27, h
EOR x1 @> 64-27, h
EOR x1 @> 64-27, h, h
MADD h, prime4, prime1, h
try4:
TBZ $2, len, try2
TBZ $2, n, try2
MOVWU.P 4(p), x2
MUL prime1, x2
ROR $64-23, h
EOR x2 @> 64-23, h
EOR x2 @> 64-23, h, h
MADD h, prime3, prime2, h
try2:
TBZ $1, len, try1
TBZ $1, n, try1
MOVHU.P 2(p), x3
AND $255, x3, x1
LSR $8, x3, x2
MUL prime5, x1
ROR $64-11, h
EOR x1 @> 64-11, h
EOR x1 @> 64-11, h, h
MUL prime1, h
MUL prime5, x2
ROR $64-11, h
EOR x2 @> 64-11, h
EOR x2 @> 64-11, h, h
MUL prime1, h
try1:
TBZ $0, len, end
TBZ $0, n, finalize
MOVBU (p), x4
MUL prime5, x4
ROR $64-11, h
EOR x4 @> 64-11, h
EOR x4 @> 64-11, h, h
MUL prime1, h
end:
finalize:
EOR h >> 33, h
MUL prime2, h
EOR h >> 29, h
@ -163,24 +163,22 @@ end:
RET
// func writeBlocks(d *Digest, b []byte) int
//
// Assumes len(b) >= 32.
TEXT ·writeBlocks(SB), NOFRAME+NOSPLIT, $0-40
LDP primes<>(SB), (prime1, prime2)
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
LDP ·primes+0(SB), (prime1, prime2)
// Load state. Assume v[1-4] are stored contiguously.
MOVD d+0(FP), digest
LDP 0(digest), (v1, v2)
LDP 16(digest), (v3, v4)
LDP b_base+8(FP), (p, len)
LDP b_base+8(FP), (p, n)
blocksLoop()
blockLoop()
// Store updated state.
STP (v1, v2), 0(digest)
STP (v3, v4), 16(digest)
BIC $31, len
MOVD len, ret+32(FP)
BIC $31, n
MOVD n, ret+32(FP)
RET

2
common/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go generated vendored
View File

@ -13,4 +13,4 @@ package xxhash
func Sum64(b []byte) uint64
//go:noescape
func writeBlocks(d *Digest, b []byte) int
func writeBlocks(s *Digest, b []byte) int

19
common/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go generated vendored
View File

@ -15,10 +15,10 @@ func Sum64(b []byte) uint64 {
var h uint64
if n >= 32 {
v1 := prime1v + prime2
v1 := primes[0] + prime2
v2 := prime2
v3 := uint64(0)
v4 := -prime1v
v4 := -primes[0]
for len(b) >= 32 {
v1 = round(v1, u64(b[0:8:len(b)]))
v2 = round(v2, u64(b[8:16:len(b)]))
@ -37,19 +37,18 @@ func Sum64(b []byte) uint64 {
h += uint64(n)
i, end := 0, len(b)
for ; i+8 <= end; i += 8 {
k1 := round(0, u64(b[i:i+8:len(b)]))
for ; len(b) >= 8; b = b[8:] {
k1 := round(0, u64(b[:8]))
h ^= k1
h = rol27(h)*prime1 + prime4
}
if i+4 <= end {
h ^= uint64(u32(b[i:i+4:len(b)])) * prime1
if len(b) >= 4 {
h ^= uint64(u32(b[:4])) * prime1
h = rol23(h)*prime2 + prime3
i += 4
b = b[4:]
}
for ; i < end; i++ {
h ^= uint64(b[i]) * prime5
for ; len(b) > 0; b = b[1:] {
h ^= uint64(b[0]) * prime5
h = rol11(h) * prime1
}

28
common/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s generated vendored
View File

@ -320,10 +320,6 @@ error_not_enough_literals:
MOVQ $0x00000004, ret+24(FP)
RET
// Return with not enough output space error
MOVQ $0x00000005, ret+24(FP)
RET
// func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// Requires: CMOV
TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32
@ -617,10 +613,6 @@ error_not_enough_literals:
MOVQ $0x00000004, ret+24(FP)
RET
// Return with not enough output space error
MOVQ $0x00000005, ret+24(FP)
RET
// func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// Requires: BMI, BMI2, CMOV
TEXT ·sequenceDecs_decode_bmi2(SB), $8-32
@ -897,10 +889,6 @@ error_not_enough_literals:
MOVQ $0x00000004, ret+24(FP)
RET
// Return with not enough output space error
MOVQ $0x00000005, ret+24(FP)
RET
// func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// Requires: BMI, BMI2, CMOV
TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32
@ -1152,10 +1140,6 @@ error_not_enough_literals:
MOVQ $0x00000004, ret+24(FP)
RET
// Return with not enough output space error
MOVQ $0x00000005, ret+24(FP)
RET
// func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool
// Requires: SSE
TEXT ·sequenceDecs_executeSimple_amd64(SB), $8-9
@ -1389,8 +1373,7 @@ loop_finished:
MOVQ ctx+0(FP), AX
MOVQ DX, 24(AX)
MOVQ DI, 104(AX)
MOVQ 80(AX), CX
SUBQ CX, SI
SUBQ 80(AX), SI
MOVQ SI, 112(AX)
RET
@ -1402,8 +1385,7 @@ error_match_off_too_big:
MOVQ ctx+0(FP), AX
MOVQ DX, 24(AX)
MOVQ DI, 104(AX)
MOVQ 80(AX), CX
SUBQ CX, SI
SUBQ 80(AX), SI
MOVQ SI, 112(AX)
RET
@ -1747,8 +1729,7 @@ loop_finished:
MOVQ ctx+0(FP), AX
MOVQ DX, 24(AX)
MOVQ DI, 104(AX)
MOVQ 80(AX), CX
SUBQ CX, SI
SUBQ 80(AX), SI
MOVQ SI, 112(AX)
RET
@ -1760,8 +1741,7 @@ error_match_off_too_big:
MOVQ ctx+0(FP), AX
MOVQ DX, 24(AX)
MOVQ DI, 104(AX)
MOVQ 80(AX), CX
SUBQ CX, SI
SUBQ 80(AX), SI
MOVQ SI, 112(AX)
RET

30
common/vendor/github.com/klauspost/compress/zstd/zstd.go generated vendored
View File

@ -36,9 +36,6 @@ const forcePreDef = false
// zstdMinMatch is the minimum zstd match length.
const zstdMinMatch = 3
// Reset the buffer offset when reaching this.
const bufferReset = math.MaxInt32 - MaxWindowSize
// fcsUnknown is used for unknown frame content size.
const fcsUnknown = math.MaxUint64
@ -110,26 +107,25 @@ func printf(format string, a ...interface{}) {
}
}
// matchLen returns the maximum length.
// matchLen returns the maximum common prefix length of a and b.
// a must be the shortest of the two.
// The function also returns whether all bytes matched.
func matchLen(a, b []byte) int {
b = b[:len(a)]
for i := 0; i < len(a)-7; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
return i + (bits.TrailingZeros64(diff) >> 3)
func matchLen(a, b []byte) (n int) {
for ; len(a) >= 8 && len(b) >= 8; a, b = a[8:], b[8:] {
diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b)
if diff != 0 {
return n + bits.TrailingZeros64(diff)>>3
}
n += 8
}
checked := (len(a) >> 3) << 3
a = a[checked:]
b = b[checked:]
for i := range a {
if a[i] != b[i] {
return i + checked
break
}
n++
}
return len(a) + checked
return n
}
func load3232(b []byte, i int32) uint32 {
@ -140,10 +136,6 @@ func load6432(b []byte, i int32) uint64 {
return binary.LittleEndian.Uint64(b[i:])
}
func load64(b []byte, i int) uint64 {
return binary.LittleEndian.Uint64(b[i:])
}
type byter interface {
Bytes() []byte
Len() int

3
common/vendor/github.com/ulikunitz/xz/.gitignore generated vendored
View File

@ -23,3 +23,6 @@ cmd/xb/xb
# default compression test file
enwik8*
# file generated by example
example.xz

2
common/vendor/github.com/ulikunitz/xz/LICENSE generated vendored
View File

@ -1,4 +1,4 @@
Copyright (c) 2014-2021 Ulrich Kunitz
Copyright (c) 2014-2022 Ulrich Kunitz
All rights reserved.
Redistribution and use in source and binary forms, with or without

4
common/vendor/github.com/ulikunitz/xz/README.md generated vendored
View File

@ -53,6 +53,10 @@ func main() {
}
```
## Documentation
You can find the full documentation at [pkg.go.dev](https://pkg.go.dev/github.com/ulikunitz/xz).
## Using the gxz compression tool
The package includes a gxz command line utility for compression and

11
common/vendor/github.com/ulikunitz/xz/TODO.md generated vendored
View File

@ -86,11 +86,20 @@
## Log
### 2022-12-12
Matt Dantay (@bodgit) reported an issue with the LZMA reader. The implementation
returned an error if the dictionary size was less than 4096 byte, but the
recommendation stated the actual used window size should be set to 4096 byte in
that case. It actually was the pull request
[#52](https://github.com/ulikunitz/xz/pull/52). The new patch v0.5.11 will fix
it.
### 2021-02-02
Mituo Heijo has fuzzed xz and found a bug in the function readIndexBody. The
function allocated a slice of records immediately after reading the value
without further checks. Since the number has been too large the make function
without further checks. Sincex the number has been too large the make function
did panic. The fix is to check the number against the expected number of records
before allocating the records.

2
common/vendor/github.com/ulikunitz/xz/bits.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/crc.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/format.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/internal/hash/doc.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/internal/hash/roller.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

5
common/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
@ -31,8 +31,7 @@ import (
// printed. There is no control over the order of the items printed and
// the format. The full format is:
//
// 2009-01-23 01:23:23.123123 /a/b/c/d.go:23: message
//
// 2009-01-23 01:23:23.123123 /a/b/c/d.go:23: message
const (
Ldate = 1 << iota // the date: 2009-01-23
Ltime // the time: 01:23:23

2
common/vendor/github.com/ulikunitz/xz/lzma/bintree.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/bitops.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/breader.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/buffer.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/decoder.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/directcodec.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/distcodec.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/encoder.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/hashtable.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/header.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/header2.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

5
common/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
@ -59,8 +59,7 @@ func (lc *lengthCodec) init() {
// Encode encodes the length offset. The length offset l can be compute by
// subtracting minMatchLen (2) from the actual length.
//
// l = length - minMatchLen
//
// l = length - minMatchLen
func (lc *lengthCodec) Encode(e *rangeEncoder, l uint32, posState uint32,
) (err error) {
if l > maxMatchLen-minMatchLen {

2
common/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/operation.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/prob.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/properties.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

4
common/vendor/github.com/ulikunitz/xz/lzma/reader.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
@ -70,7 +70,7 @@ func (c ReaderConfig) NewReader(lzma io.Reader) (r *Reader, err error) {
return nil, err
}
if r.h.dictCap < MinDictCap {
return nil, errors.New("lzma: dictionary capacity too small")
r.h.dictCap = MinDictCap
}
dictCap := r.h.dictCap
if c.DictCap > dictCap {

2
common/vendor/github.com/ulikunitz/xz/lzma/reader2.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/state.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/writer.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzma/writer2.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/lzmafilter.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/none-check.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/reader.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

2
common/vendor/github.com/ulikunitz/xz/writer.go generated vendored
View File

@ -1,4 +1,4 @@
// Copyright 2014-2021 Ulrich Kunitz. All rights reserved.
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

6
common/vendor/modules.txt vendored
View File

@ -157,7 +157,7 @@ github.com/containers/ocicrypt/keywrap/pkcs7
github.com/containers/ocicrypt/spec
github.com/containers/ocicrypt/utils
github.com/containers/ocicrypt/utils/keyprovider
# github.com/containers/storage v1.44.1-0.20221209084436-73d739442168
# github.com/containers/storage v1.44.1-0.20230105105526-fc91849352e5
## explicit; go 1.17
github.com/containers/storage
github.com/containers/storage/drivers
@ -321,7 +321,7 @@ github.com/jinzhu/copier
# github.com/json-iterator/go v1.1.12
## explicit; go 1.12
github.com/json-iterator/go
# github.com/klauspost/compress v1.15.12
# github.com/klauspost/compress v1.15.14
## explicit; go 1.17
github.com/klauspost/compress
github.com/klauspost/compress/flate
@ -494,7 +494,7 @@ github.com/theupdateframework/go-tuf/encrypted
# github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399
## explicit
github.com/titanous/rocacheck
# github.com/ulikunitz/xz v0.5.10
# github.com/ulikunitz/xz v0.5.11
## explicit; go 1.12
github.com/ulikunitz/xz
github.com/ulikunitz/xz/internal/hash