containerd
/
fuse-overlayfs-snapshotter
mirror of https://github.com/containerd/fuse-overlayfs-snapshotter.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

rebase 

rebased with containerd/containerd@f01665aa02

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
This commit is contained in:
Akihiro Suda 2019-12-12 15:32:28 +09:00
parent 0189d895f6
commit 2c1beabab7
16 changed files with 228 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -1,4 +1,4 @@
ARG FUSEOVERLAYFS_COMMIT=ae96c48f1a4f1a8c9f7170238b31c8246ce34b84
ARG FUSEOVERLAYFS_COMMIT=v0.7.2
ARG ROOTLESSKIT_COMMIT=v0.7.0
ARG SHADOW_COMMIT=4.7

2
README.md
View File

@ -6,7 +6,7 @@ Unlike `overlayfs`, `fuse-overlayfs` can be used as a non-root user without patc
## Requirements
* kernel >= 4.18 (So this can't be tested in Travis CI, which uses kernel 4.15, as of October 2019)
* containerd with [PR #3765](https://github.com/containerd/containerd/pull/3765)
* fuse-overlayfs >= [20191020](https://github.com/containers/fuse-overlayfs/commit/c9bbc94ab65467481ea0e0810eea8fd1bfd8a4bf)
* fuse-overlayfs >= v0.7.0
## How to test

4
fuseoverlayfs.go
View File

@ -287,13 +287,13 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) {
}
// Walk the committed snapshots.
func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error {
func (o *snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
ctx, t, err := o.ms.TransactionContext(ctx, false)
if err != nil {
return err
}
defer t.Rollback()
return storage.WalkInfo(ctx, fn)
return storage.WalkInfo(ctx, fn, fs...)
}
// Cleanup cleans up disk resources from removed or abandoned snapshots

2
go.mod
View File

@ -24,4 +24,4 @@ require (
)
// https://github.com/containerd/containerd/pull/3765
replace github.com/containerd/containerd => github.com/AkihiroSuda/containerd v1.1.1-0.20191020052632-e9c211fd89d2
replace github.com/containerd/containerd => github.com/AkihiroSuda/containerd v1.1.1-0.20191212062816-cbf52e6f9154

10
go.sum
View File

@ -1,12 +1,6 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/AkihiroSuda/containerd v1.1.1-0.20191019163216-54a33eec6760 h1:Os/M4cP9SGyzYitUSyhXrFk0ogfIeKTBzkR5Klf5F+g=
github.com/AkihiroSuda/containerd v1.1.1-0.20191019163216-54a33eec6760/go.mod h1:5IBP++IFtudvjLCXBrBPdXGu8s/AL9xiPCjz0K9psr4=
github.com/AkihiroSuda/containerd v1.1.1-0.20191019164352-a6dd6aa708ab h1:DdLvkgW4fnn4t2G6iUqgzJN7CQIRWh/X//U9sUBFdfg=
github.com/AkihiroSuda/containerd v1.1.1-0.20191019164352-a6dd6aa708ab/go.mod h1:5IBP++IFtudvjLCXBrBPdXGu8s/AL9xiPCjz0K9psr4=
github.com/AkihiroSuda/containerd v1.1.1-0.20191020050244-0469cacda076 h1:uakXWILS0/ZI/XFySskOnVAlmrw8LbVN/gqpVATzfBc=
github.com/AkihiroSuda/containerd v1.1.1-0.20191020050244-0469cacda076/go.mod h1:5IBP++IFtudvjLCXBrBPdXGu8s/AL9xiPCjz0K9psr4=
github.com/AkihiroSuda/containerd v1.1.1-0.20191020052632-e9c211fd89d2 h1:P1crhhX1pWwIi4cOLpwUNVLu3EpakfwnRaeUWzjEibg=
github.com/AkihiroSuda/containerd v1.1.1-0.20191020052632-e9c211fd89d2/go.mod h1:5IBP++IFtudvjLCXBrBPdXGu8s/AL9xiPCjz0K9psr4=
github.com/AkihiroSuda/containerd v1.1.1-0.20191212062816-cbf52e6f9154 h1:sG0j7H0xn9w4nbLDzUe7tWLUTMGj2sj6Apsi1zB/FxI=
github.com/AkihiroSuda/containerd v1.1.1-0.20191212062816-cbf52e6f9154/go.mod h1:5IBP++IFtudvjLCXBrBPdXGu8s/AL9xiPCjz0K9psr4=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=

2
vendor/github.com/containerd/containerd/events/exchange/exchange.go generated vendored
View File

@ -225,7 +225,7 @@ func validateTopic(topic string) error {
}
func validateEnvelope(envelope *events.Envelope) error {
if err := namespaces.Validate(envelope.Namespace); err != nil {
if err := identifiers.Validate(envelope.Namespace); err != nil {
return errors.Wrapf(err, "event envelope has invalid namespace")
}

10
vendor/github.com/containerd/containerd/identifiers/validate.go generated vendored
View File

@ -42,13 +42,13 @@ var (
identifierRe = regexp.MustCompile(reAnchor(alphanum + reGroup(separators+reGroup(alphanum)) + "*"))
)
// Validate return nil if the string s is a valid identifier.
// Validate returns nil if the string s is a valid identifier.
//
// identifiers must be valid domain names according to RFC 1035, section 2.3.1. To
// enforce case insensitivity, all characters must be lower case.
// identifiers are similar to the domain name rules according to RFC 1035, section 2.3.1. However
// rules in this package are relaxed to allow numerals to follow period (".") and mixed case is
// allowed.
//
// In general, identifiers that pass this validation, should be safe for use as
// a domain names or filesystem path component.
// In general identifiers that pass this validation should be safe for use as filesystem path components.
func Validate(s string) error {
if len(s) == 0 {
return errors.Wrapf(errdefs.ErrInvalidArgument, "identifier must not be empty")

30
vendor/github.com/containerd/containerd/log/context.go generated vendored
View File

@ -18,7 +18,6 @@ package log
import (
"context"
"sync/atomic"
"github.com/sirupsen/logrus"
)
@ -38,23 +37,10 @@ type (
loggerKey struct{}
)
// TraceLevel is the log level for tracing. Trace level is lower than debug level,
// and is usually used to trace detailed behavior of the program.
const TraceLevel = logrus.Level(uint32(logrus.DebugLevel + 1))
// RFC3339NanoFixed is time.RFC3339Nano with nanoseconds padded using zeros to
// ensure the formatted time is always the same number of characters.
const RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
// ParseLevel takes a string level and returns the Logrus log level constant.
// It supports trace level.
func ParseLevel(lvl string) (logrus.Level, error) {
if lvl == "trace" {
return TraceLevel, nil
}
return logrus.ParseLevel(lvl)
}
// WithLogger returns a new context with the provided logger. Use in
// combination with logger.WithField(s) for great effect.
func WithLogger(ctx context.Context, logger *logrus.Entry) context.Context {
@ -72,19 +58,3 @@ func GetLogger(ctx context.Context) *logrus.Entry {
return logger.(*logrus.Entry)
}
// Trace logs a message at level Trace with the log entry passed-in.
func Trace(e *logrus.Entry, args ...interface{}) {
level := logrus.Level(atomic.LoadUint32((*uint32)(&e.Logger.Level)))
if level >= TraceLevel {
e.Debug(args...)
}
}
// Tracef logs a message at level Trace with the log entry passed-in.
func Tracef(e *logrus.Entry, format string, args ...interface{}) {
level := logrus.Level(atomic.LoadUint32((*uint32)(&e.Logger.Level)))
if level >= TraceLevel {
e.Debugf(format, args...)
}
}

16
vendor/github.com/containerd/containerd/mount/mount_linux.go generated vendored
View File

@ -107,6 +107,22 @@ func Unmount(target string, flags int) error {
}
func unmount(target string, flags int) error {
inf, err := Lookup(target)
if err != nil {
// UnmountAll() expects unix.EINVAL
return unix.EINVAL
}
// For FUSE mounts, attempting to execute fusermount helper binary is preferred
// https://github.com/containerd/containerd/pull/3765#discussion_r342083514
if strings.HasPrefix(inf.FSType, "fuse3.") || strings.HasPrefix(inf.FSType, "fuse.") {
for _, helperBinary := range []string{"fusermount3", "fusermount"} {
cmd := exec.Command(helperBinary, "-u", target)
if err := cmd.Run(); err == nil {
return nil
}
// ignore error and try unix.Unmount
}
}
for i := 0; i < 50; i++ {
if err := unix.Unmount(target, flags); err != nil {
switch err {

3
vendor/github.com/containerd/containerd/namespaces/context.go generated vendored
View File

@ -21,6 +21,7 @@ import (
"os"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/identifiers"
"github.com/pkg/errors"
)
@ -70,7 +71,7 @@ func NamespaceRequired(ctx context.Context) (string, error) {
if !ok || namespace == "" {
return "", errors.Wrapf(errdefs.ErrFailedPrecondition, "namespace is required")
}
if err := Validate(namespace); err != nil {
if err := identifiers.Validate(namespace); err != nil {
return "", errors.Wrap(err, "namespace validation")
}
return namespace, nil

83
vendor/github.com/containerd/containerd/namespaces/validate.go generated vendored
View File

@ -1,83 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package namespaces provides tools for working with namespaces across
// containerd.
//
// Namespaces collect resources such as containers and images, into a unique
// identifier space. This means that two applications can use the same
// identifiers and not conflict while using containerd.
//
// This package can be used to ensure that client and server functions
// correctly store the namespace on the context.
package namespaces
import (
"regexp"
"github.com/containerd/containerd/errdefs"
"github.com/pkg/errors"
)
const (
maxLength = 76
alpha = `[A-Za-z]`
alphanum = `[A-Za-z0-9]+`
label = alpha + alphanum + `(:?[-]+` + alpha + alphanum + `)*`
)
var (
// namespaceRe validates that a namespace matches valid identifiers.
//
// Rules for domains, defined in RFC 1035, section 2.3.1, are used for
// namespaces.
namespaceRe = regexp.MustCompile(reAnchor(label + reGroup("[.]"+reGroup(label)) + "*"))
)
// Validate returns nil if the string s is a valid namespace.
//
// To allow such namespace identifiers to be used across various contexts
// safely, the character set has been restricted to that defined for domains in
// RFC 1035, section 2.3.1. This will make namespace identifiers safe for use
// across networks, filesystems and other media.
//
// The identifier specification departs from RFC 1035 in that it allows
// "labels" to start with number and only enforces a total length restriction
// of 76 characters.
//
// While the character set may be expanded in the future, namespace identifiers
// are guaranteed to be safely used as filesystem path components.
//
// For the most part, this doesn't need to be called directly when using the
// context-oriented functions.
func Validate(s string) error {
if len(s) > maxLength {
return errors.Wrapf(errdefs.ErrInvalidArgument, "namespace %q greater than maximum length (%d characters)", s, maxLength)
}
if !namespaceRe.MatchString(s) {
return errors.Wrapf(errdefs.ErrInvalidArgument, "namespace %q must match %v", s, namespaceRe)
}
return nil
}
func reGroup(s string) string {
return `(?:` + s + `)`
}
func reAnchor(s string) string {
return `^` + s + `$`
}

12
vendor/github.com/containerd/containerd/platforms/cpuinfo.go generated vendored
View File

@ -96,16 +96,16 @@ func getCPUVariant() string {
return ""
}
switch variant {
case "8", "AArch64":
switch strings.ToLower(variant) {
case "8", "aarch64":
variant = "v8"
case "7", "7M", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)":
case "7", "7m", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)":
variant = "v7"
case "6", "6TEJ":
case "6", "6tej":
variant = "v6"
case "5", "5T", "5TE", "5TEJ":
case "5", "5t", "5te", "5tej":
variant = "v5"
case "4", "4T":
case "4", "4t":
variant = "v4"
case "3":
variant = "v3"

15
vendor/github.com/containerd/containerd/snapshots/snapshotter.go generated vendored
View File

@ -118,6 +118,9 @@ func (u *Usage) Add(other Usage) {
u.Inodes += other.Inodes
}
// WalkFunc defines the callback for a snapshot walk.
type WalkFunc func(context.Context, Info) error
// Snapshotter defines the methods required to implement a snapshot snapshotter for
// allocating, snapshotting and mounting filesystem changesets. The model works
// by building up sets of changes with parent-child relationships.
@ -314,9 +317,15 @@ type Snapshotter interface {
// removed before proceeding.
Remove(ctx context.Context, key string) error
// Walk all snapshots in the snapshotter. For each snapshot in the
// snapshotter, the function will be called.
Walk(ctx context.Context, fn func(context.Context, Info) error) error
// Walk will call the provided function for each snapshot in the
// snapshotter which match the provided filters. If no filters are
// given all items will be walked.
// Filters:
// name
// parent
// kind (active,view,committed)
// labels.(label)
Walk(ctx context.Context, fn WalkFunc, filters ...string) error
// Close releases the internal resources.
//

46
vendor/github.com/containerd/containerd/snapshots/storage/bolt.go generated vendored
View File

@ -24,6 +24,7 @@ import (
"time"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/filters"
"github.com/containerd/containerd/metadata/boltutil"
"github.com/containerd/containerd/snapshots"
"github.com/pkg/errors"
@ -144,7 +145,12 @@ func UpdateInfo(ctx context.Context, info snapshots.Info, fieldpaths ...string)
// WalkInfo iterates through all metadata Info for the stored snapshots and
// calls the provided function for each. Requires a context with a storage
// transaction.
func WalkInfo(ctx context.Context, fn func(context.Context, snapshots.Info) error) error {
func WalkInfo(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error {
filter, err := filters.ParseAll(fs...)
if err != nil {
return err
}
// TODO: allow indexes (name, parent, specific labels)
return withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
return bkt.ForEach(func(k, v []byte) error {
// skip non buckets
@ -160,6 +166,9 @@ func WalkInfo(ctx context.Context, fn func(context.Context, snapshots.Info) erro
if err := readSnapshot(sbkt, nil, &si); err != nil {
return err
}
if !filter.Match(adaptSnapshot(si)) {
return nil
}
return fn(ctx, si)
})
@ -297,7 +306,7 @@ func Remove(ctx context.Context, key string) (string, snapshots.Kind, error) {
}
if err := readSnapshot(sbkt, &id, &si); err != nil {
errors.Wrapf(err, "failed to read snapshot %s", key)
return errors.Wrapf(err, "failed to read snapshot %s", key)
}
if pbkt != nil {
@ -604,3 +613,36 @@ func encodeID(id uint64) ([]byte, error) {
}
return idEncoded, nil
}
func adaptSnapshot(info snapshots.Info) filters.Adaptor {
return filters.AdapterFunc(func(fieldpath []string) (string, bool) {
if len(fieldpath) == 0 {
return "", false
}
switch fieldpath[0] {
case "kind":
switch info.Kind {
case snapshots.KindActive:
return "active", true
case snapshots.KindView:
return "view", true
case snapshots.KindCommitted:
return "committed", true
}
case "name":
return info.Name, true
case "parent":
return info.Parent, true
case "labels":
if len(info.Labels) == 0 {
return "", false
}
v, ok := info.Labels[strings.Join(fieldpath[1:], ".")]
return v, ok
}
return "", false
})
}

134
vendor/github.com/containerd/containerd/snapshots/testsuite/testsuite.go generated vendored
View File

@ -23,6 +23,7 @@ import (
"math/rand"
"os"
"path/filepath"
"sort"
"testing"
"time"
@ -49,6 +50,7 @@ func SnapshotterSuite(t *testing.T, name string, snapshotterFn func(ctx context.
t.Run("PreareViewFailingtest", makeTest(name, snapshotterFn, checkSnapshotterPrepareView))
t.Run("Update", makeTest(name, snapshotterFn, checkUpdate))
t.Run("Remove", makeTest(name, snapshotterFn, checkRemove))
t.Run("Walk", makeTest(name, snapshotterFn, checkWalk))
t.Run("LayerFileupdate", makeTest(name, snapshotterFn, checkLayerFileUpdate))
t.Run("RemoveDirectoryInLowerLayer", makeTest(name, snapshotterFn, checkRemoveDirectoryInLowerLayer))
@ -961,3 +963,135 @@ func check128LayersMount(name string) func(ctx context.Context, t *testing.T, sn
}
}
}
func checkWalk(ctx context.Context, t *testing.T, snapshotter snapshots.Snapshotter, work string) {
opt := snapshots.WithLabels(map[string]string{
"containerd.io/gc.root": "check-walk",
})
// No parent active
if _, err := snapshotter.Prepare(ctx, "a-np", "", opt); err != nil {
t.Fatal(err)
}
// Base parent
if _, err := snapshotter.Prepare(ctx, "p-tmp", "", opt); err != nil {
t.Fatal(err)
}
if err := snapshotter.Commit(ctx, "p", "p-tmp", opt); err != nil {
t.Fatal(err)
}
// Active
if _, err := snapshotter.Prepare(ctx, "a", "p", opt); err != nil {
t.Fatal(err)
}
// View
if _, err := snapshotter.View(ctx, "v", "p", opt); err != nil {
t.Fatal(err)
}
// Base parent with label=1
if _, err := snapshotter.Prepare(ctx, "p-wl-tmp", "", opt); err != nil {
t.Fatal(err)
}
if err := snapshotter.Commit(ctx, "p-wl", "p-wl-tmp", snapshots.WithLabels(map[string]string{
"l": "1",
"containerd.io/gc.root": "check-walk",
})); err != nil {
t.Fatal(err)
}
// active with label=2
if _, err := snapshotter.Prepare(ctx, "a-wl", "p-wl", snapshots.WithLabels(map[string]string{
"l": "2",
"containerd.io/gc.root": "check-walk",
})); err != nil {
t.Fatal(err)
}
// view with label=3
if _, err := snapshotter.View(ctx, "v-wl", "p-wl", snapshots.WithLabels(map[string]string{
"l": "3",
"containerd.io/gc.root": "check-walk",
})); err != nil {
t.Fatal(err)
}
// no parent active with label=2
if _, err := snapshotter.Prepare(ctx, "a-np-wl", "", snapshots.WithLabels(map[string]string{
"l": "2",
"containerd.io/gc.root": "check-walk",
})); err != nil {
t.Fatal(err)
}
for i, tc := range []struct {
matches []string
filters []string
}{
{
matches: []string{"a-np", "p", "a", "v", "p-wl", "a-wl", "v-wl", "a-np-wl"},
filters: []string{"labels.\"containerd.io/gc.root\"==check-walk"},
},
{
matches: []string{"a-np", "a", "a-wl", "a-np-wl"},
filters: []string{"kind==active,labels.\"containerd.io/gc.root\"==check-walk"},
},
{
matches: []string{"v", "v-wl"},
filters: []string{"kind==view,labels.\"containerd.io/gc.root\"==check-walk"},
},
{
matches: []string{"p", "p-wl"},
filters: []string{"kind==committed,labels.\"containerd.io/gc.root\"==check-walk"},
},
{
matches: []string{"p", "a-np-wl"},
filters: []string{"name==p", "name==a-np-wl"},
},
{
matches: []string{"a-wl"},
filters: []string{"name==a-wl,labels.l"},
},
{
matches: []string{"a", "v"},
filters: []string{"parent==p"},
},
{
matches: []string{"a", "v", "a-wl", "v-wl"},
filters: []string{"parent!=\"\",labels.\"containerd.io/gc.root\"==check-walk"},
},
{
matches: []string{"p-wl", "a-wl", "v-wl", "a-np-wl"},
filters: []string{"labels.l"},
},
{
matches: []string{"a-wl", "a-np-wl"},
filters: []string{"labels.l==2"},
},
} {
actual := []string{}
err := snapshotter.Walk(ctx, func(ctx context.Context, si snapshots.Info) error {
actual = append(actual, si.Name)
return nil
}, tc.filters...)
if err != nil {
t.Fatal(err)
}
sort.Strings(tc.matches)
sort.Strings(actual)
if len(actual) != len(tc.matches) {
t.Errorf("[%d] Unexpected result (size):\nActual:\n\t%#v\nExpected:\n\t%#v", i, actual, tc.matches)
continue
}
for j := range actual {
if actual[j] != tc.matches[j] {
t.Errorf("[%d] Unexpected result @%d:\nActual:\n\t%#vExpected:\n\t%#v", i, j, actual, tc.matches)
break
}
}
}
}

2
vendor/modules.txt vendored
View File

@ -17,7 +17,7 @@ github.com/Microsoft/hcsshim/internal/schema1
github.com/Microsoft/hcsshim/internal/schema2
github.com/Microsoft/hcsshim/internal/timeout
github.com/Microsoft/hcsshim/internal/wclayer
# github.com/containerd/containerd v0.0.0-00010101000000-000000000000 => github.com/AkihiroSuda/containerd v1.1.1-0.20191020052632-e9c211fd89d2
# github.com/containerd/containerd v0.0.0-00010101000000-000000000000 => github.com/AkihiroSuda/containerd v1.1.1-0.20191212062816-cbf52e6f9154
github.com/containerd/containerd/errdefs
github.com/containerd/containerd/events
github.com/containerd/containerd/events/exchange