containers
/
storage
mirror of https://github.com/containers/storage.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support additional layer store 

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>
This commit is contained in:
ktock 2020-12-18 12:07:29 +09:00
parent f21e201f59
commit 64f018103e
6 changed files with 450 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
drivers/driver.go
View File

@ -14,6 +14,7 @@ import (
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/directory"
"github.com/containers/storage/pkg/idtools"
digest "github.com/opencontainers/go-digest"
)
// FsMagic unsigned id of the filesystem in use.
@ -33,7 +34,9 @@ var (
// ErrPrerequisites returned when driver does not meet prerequisites.
ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)")
// ErrIncompatibleFS returned when file system is not supported.
ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver")
ErrIncompatibleFS = errors.New("backing file system is unsupported for this graph driver")
// ErrLayerUnknown returned when the specified layer is unknown by the driver.
ErrLayerUnknown = errors.New("unknown layer")
)
//CreateOpts contains optional arguments for Create() and CreateReadWrite()
@ -117,6 +120,7 @@ type ProtoDriver interface {
// known to this driver.
Cleanup() error
// AdditionalImageStores returns additional image stores supported by the driver
// This API is experimental and can be changed without bumping the major version number.
AdditionalImageStores() []string
}
@ -180,6 +184,30 @@ type CapabilityDriver interface {
Capabilities() Capabilities
}
// AdditionalLayer reprents a layer that is stored in the additional layer store
// This API is experimental and can be changed without bumping the major version number.
type AdditionalLayer interface {
// CreateAs creates a new layer from this additional layer
CreateAs(id, parent string) error
// Info returns arbitrary information stored along with this layer (i.e. `info` file)
Info() (io.ReadCloser, error)
// Release tells the additional layer store that we don't use this handler.
Release()
}
// AdditionalLayerStoreDriver is the interface for driver that supports
// additional layer store functionality.
// This API is experimental and can be changed without bumping the major version number.
type AdditionalLayerStoreDriver interface {
Driver
// LookupAdditionalLayer looks up additional layer store by the specified
// digest and ref and returns an object representing that layer.
LookupAdditionalLayer(d digest.Digest, ref string) (AdditionalLayer, error)
}
// DiffGetterDriver is the interface for layered file system drivers that
// provide a specialized function for getting file contents for tar-split.
type DiffGetterDriver interface {

253
drivers/overlay/overlay.go
View File

@ -4,6 +4,7 @@ package overlay
import (
"bytes"
"encoding/base64"
"fmt"
"io"
"io/ioutil"
@ -31,6 +32,7 @@ import (
"github.com/containers/storage/pkg/unshare"
units "github.com/docker/go-units"
"github.com/hashicorp/go-multierror"
digest "github.com/opencontainers/go-digest"
rsystem "github.com/opencontainers/runc/libcontainer/system"
"github.com/opencontainers/selinux/go-selinux"
"github.com/opencontainers/selinux/go-selinux/label"
@ -94,6 +96,7 @@ const (
type overlayOptions struct {
imageStores []string
layerStores []additionalLayerStore
quota quota.Quota
mountProgram string
skipMountHome bool
@ -119,6 +122,17 @@ type Driver struct {
locker *locker.Locker
}
type additionalLayerStore struct {
// path is the directory where this store is available on the host.
path string
// withReference is true when the store contains image reference information (base64-encoded)
// in its layer search path so the path to the diff will be
// <path>/base64(reference)/<layerdigest>/
withReference bool
}
var (
backingFs = "<unknown>"
projectQuotaSupported = false
@ -397,6 +411,42 @@ func parseOptions(options []string) (*overlayOptions, error) {
}
o.imageStores = append(o.imageStores, store)
}
case "additionallayerstore":
logrus.Debugf("overlay: additionallayerstore=%s", val)
// Additional read only layer stores to use for lower paths
if val == "" {
continue
}
for _, lstore := range strings.Split(val, ",") {
elems := strings.Split(lstore, ":")
lstore = filepath.Clean(elems[0])
if !filepath.IsAbs(lstore) {
return nil, fmt.Errorf("overlay: additionallayerstore path %q is not absolute. Can not be relative", lstore)
}
st, err := os.Stat(lstore)
if err != nil {
return nil, errors.Wrap(err, "overlay: can't stat additionallayerstore dir")
}
if !st.IsDir() {
return nil, fmt.Errorf("overlay: additionallayerstore path %q must be a directory", lstore)
}
var withReference bool
for _, e := range elems[1:] {
switch e {
case "ref":
if withReference {
return nil, fmt.Errorf("overlay: additionallayerstore config of %q contains %q option twice", lstore, e)
}
withReference = true
default:
return nil, fmt.Errorf("overlay: additionallayerstore config %q contains unknown option %q", lstore, e)
}
}
o.layerStores = append(o.layerStores, additionalLayerStore{
path: lstore,
withReference: withReference,
})
}
case "mount_program":
logrus.Debugf("overlay: mount_program=%s", val)
if val != "" {
@ -657,6 +707,24 @@ func (d *Driver) Cleanup() error {
return mount.Unmount(d.home)
}
// LookupAdditionalLayer looks up additional layer store by the specified
// digest and ref and returns an object representing that layer.
// This API is experimental and can be changed without bumping the major version number.
func (d *Driver) LookupAdditionalLayer(dgst digest.Digest, ref string) (graphdriver.AdditionalLayer, error) {
l, err := d.getAdditionalLayerPath(dgst, ref)
if err != nil {
return nil, err
}
// Tell the additional layer store that we use this layer.
// This will increase reference counter on the store's side.
// This will be decreased on Release() method.
notifyUseAdditionalLayer(l)
return &additionalLayer{
path: l,
d: d,
}, nil
}
// CreateFromTemplate creates a layer with the same contents and parent as another layer.
func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error {
if readWrite {
@ -959,6 +1027,8 @@ func (d *Driver) Remove(id string) error {
}
}
d.releaseAdditionalLayerByID(id)
if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) {
return err
}
@ -1422,7 +1492,10 @@ func (f fileGetNilCloser) Close() error {
// DiffGetter returns a FileGetCloser that can read files from the directory that
// contains files for the layer differences. Used for direct access for tar-split.
func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) {
p := d.getDiffPath(id)
p, err := d.getDiffPath(id)
if err != nil {
return nil, err
}
return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil
}
@ -1444,7 +1517,10 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts)
idMappings = &idtools.IDMappings{}
}
applyDir := d.getDiffPath(id)
applyDir, err := d.getDiffPath(id)
if err != nil {
return 0, err
}
logrus.Debugf("Applying tar in %s", applyDir)
// Overlay doesn't need the parent id to apply the diff
@ -1462,10 +1538,23 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts)
return directory.Size(applyDir)
}
func (d *Driver) getDiffPath(id string) string {
func (d *Driver) getDiffPath(id string) (string, error) {
dir := d.dir(id)
return redirectDiffIfAdditionalLayer(path.Join(dir, "diff"))
}
return path.Join(dir, "diff")
func (d *Driver) getLowerDiffPaths(id string) ([]string, error) {
layers, err := d.getLowerDirs(id)
if err != nil {
return nil, err
}
for i, l := range layers {
layers[i], err = redirectDiffIfAdditionalLayer(l)
if err != nil {
return nil, err
}
}
return layers, nil
}
// DiffSize calculates the changes between the specified id
@ -1476,7 +1565,11 @@ func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent stri
return d.naiveDiff.DiffSize(id, idMappings, parent, parentMappings, mountLabel)
}
return directory.Size(d.getDiffPath(id))
p, err := d.getDiffPath(id)
if err != nil {
return 0, err
}
return directory.Size(p)
}
// Diff produces an archive of the changes between the specified
@ -1490,12 +1583,15 @@ func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string,
idMappings = &idtools.IDMappings{}
}
lowerDirs, err := d.getLowerDirs(id)
lowerDirs, err := d.getLowerDiffPaths(id)
if err != nil {
return nil, err
}
diffPath := d.getDiffPath(id)
diffPath, err := d.getDiffPath(id)
if err != nil {
return nil, err
}
logrus.Debugf("Tar with options on %s", diffPath)
return archive.TarWithOptions(diffPath, &archive.TarOptions{
Compression: archive.Uncompressed,
@ -1514,8 +1610,11 @@ func (d *Driver) Changes(id string, idMappings *idtools.IDMappings, parent strin
}
// Overlay doesn't have snapshots, so we need to get changes from all parent
// layers.
diffPath := d.getDiffPath(id)
layers, err := d.getLowerDirs(id)
diffPath, err := d.getDiffPath(id)
if err != nil {
return nil, err
}
layers, err := d.getLowerDiffPaths(id)
if err != nil {
return nil, err
}
@ -1627,3 +1726,139 @@ func nameWithSuffix(name string, number int) string {
}
return fmt.Sprintf("%s%d", name, number)
}
func (d *Driver) getAdditionalLayerPath(dgst digest.Digest, ref string) (string, error) {
refElem := base64.StdEncoding.EncodeToString([]byte(ref))
for _, ls := range d.options.layerStores {
ref := ""
if ls.withReference {
ref = refElem
}
target := path.Join(ls.path, ref, dgst.String())
// Check if all necessary files exist
for _, p := range []string{
filepath.Join(target, "diff"),
filepath.Join(target, "info"),
// TODO(ktock): We should have an API to expose the stream data of this layer
// to enable the client to retrieve the entire contents of this
// layer when it exports this layer.
} {
if _, err := os.Stat(p); err != nil {
return "", errors.Wrapf(graphdriver.ErrLayerUnknown,
"failed to stat additional layer %q: %v", p, err)
}
}
return target, nil
}
return "", errors.Wrapf(graphdriver.ErrLayerUnknown,
"additional layer (%q, %q) not found", dgst, ref)
}
func (d *Driver) releaseAdditionalLayerByID(id string) {
if al, err := ioutil.ReadFile(path.Join(d.dir(id), "additionallayer")); err == nil {
notifyReleaseAdditionalLayer(string(al))
} else if !os.IsNotExist(err) {
logrus.Warnf("unexpected error on reading Additional Layer Store pointer %v", err)
}
}
// additionalLayer represents a layer in Additional Layer Store.
type additionalLayer struct {
path string
d *Driver
releaseOnce sync.Once
}
// Info returns arbitrary information stored along with this layer (i.e. `info` file).
// This API is experimental and can be changed without bumping the major version number.
func (al *additionalLayer) Info() (io.ReadCloser, error) {
return os.Open(filepath.Join(al.path, "info"))
}
// CreateAs creates a new layer from this additional layer.
// This API is experimental and can be changed without bumping the major version number.
func (al *additionalLayer) CreateAs(id, parent string) error {
// TODO: support opts
if err := al.d.Create(id, parent, nil); err != nil {
return err
}
dir := al.d.dir(id)
diffDir := path.Join(dir, "diff")
if err := os.RemoveAll(diffDir); err != nil {
return err
}
// tell the additional layer store that we use this layer.
// mark this layer as "additional layer"
if err := ioutil.WriteFile(path.Join(dir, "additionallayer"), []byte(al.path), 0644); err != nil {
return err
}
notifyUseAdditionalLayer(al.path)
return os.Symlink(filepath.Join(al.path, "diff"), diffDir)
}
// Release tells the additional layer store that we don't use this handler.
// This API is experimental and can be changed without bumping the major version number.
func (al *additionalLayer) Release() {
// Tell the additional layer store that we don't use this layer handler.
// This will decrease the reference counter on the store's side, which was
// increased in LookupAdditionalLayer (so this must be called only once).
al.releaseOnce.Do(func() {
notifyReleaseAdditionalLayer(al.path)
})
}
// notifyUseAdditionalLayer notifies Additional Layer Store that we use the specified layer.
// This is done by creating "use" file in the layer directory. This is useful for
// Additional Layer Store to consider when to perform GC. Notification-aware Additional
// Layer Store must return ENOENT.
func notifyUseAdditionalLayer(al string) {
if !path.IsAbs(al) {
logrus.Warnf("additionallayer must be absolute (got: %v)", al)
return
}
useFile := path.Join(al, "use")
f, err := os.Create(useFile)
if os.IsNotExist(err) {
return
} else if err == nil {
f.Close()
if err := os.Remove(useFile); err != nil {
logrus.Warnf("failed to remove use file")
}
}
logrus.Warnf("unexpected error by Additional Layer Store %v during use; GC doesn't seem to be supported", err)
}
// notifyReleaseAdditionalLayer notifies Additional Layer Store that we don't use the specified
// layer anymore. This is done by rmdir-ing the layer directory. This is useful for
// Additional Layer Store to consider when to perform GC. Notification-aware Additional
// Layer Store must return ENOENT.
func notifyReleaseAdditionalLayer(al string) {
if !path.IsAbs(al) {
logrus.Warnf("additionallayer must be absolute (got: %v)", al)
return
}
// tell the additional layer store that we don't use this layer anymore.
err := unix.Rmdir(al)
if os.IsNotExist(err) {
return
}
logrus.Warnf("unexpected error by Additional Layer Store %v during release; GC doesn't seem to be supported", err)
}
// redirectDiffIfAdditionalLayer checks if the passed diff path is Additional Layer and
// returns the redirected path. If the passed diff is not the one in Additional Layer
// Store, it returns the original path without changes.
func redirectDiffIfAdditionalLayer(diffPath string) (string, error) {
if ld, err := os.Readlink(diffPath); err == nil {
// diff is the link to Additional Layer Store
if !path.IsAbs(ld) {
return "", fmt.Errorf("linkpath must be absolute (got: %q)", ld)
}
diffPath = ld
} else if err.(*os.PathError).Err != syscall.EINVAL {
return "", err
}
return diffPath, nil
}

57
layers.go
View File

@ -250,6 +250,11 @@ type LayerStore interface {
// LoadLocked wraps Load in a locked state. This means it loads the store
// and cleans-up invalid layers if needed.
LoadLocked() error
// PutAdditionalLayer creates a layer using the diff contained in the additional layer
// store.
// This API is experimental and can be changed without bumping the major version number.
PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error)
}
type layerStore struct {
@ -610,6 +615,58 @@ func (r *layerStore) Status() ([][2]string, error) {
return r.driver.Status(), nil
}
func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) {
if duplicateLayer, idInUse := r.byid[id]; idInUse {
return duplicateLayer, ErrDuplicateID
}
for _, name := range names {
if _, nameInUse := r.byname[name]; nameInUse {
return nil, ErrDuplicateName
}
}
parent := ""
if parentLayer != nil {
parent = parentLayer.ID
}
info, err := aLayer.Info()
if err != nil {
return nil, err
}
defer info.Close()
layer = &Layer{}
if err := json.NewDecoder(info).Decode(layer); err != nil {
return nil, err
}
layer.ID = id
layer.Parent = parent
layer.Created = time.Now().UTC()
if err := aLayer.CreateAs(id, parent); err != nil {
return nil, err
}
// TODO: check if necessary fields are filled
r.layers = append(r.layers, layer)
r.idindex.Add(id)
r.byid[id] = layer
for _, name := range names { // names got from the additional layer store won't be used
r.byname[name] = layer
}
if layer.CompressedDigest != "" {
r.bycompressedsum[layer.CompressedDigest] = append(r.bycompressedsum[layer.CompressedDigest], layer.ID)
}
if layer.UncompressedDigest != "" {
r.byuncompressedsum[layer.CompressedDigest] = append(r.byuncompressedsum[layer.CompressedDigest], layer.ID)
}
if err := r.Save(); err != nil {
r.driver.Remove(id)
return nil, err
}
return copyLayer(layer), nil
}
func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (layer *Layer, size int64, err error) {
if !r.IsReadWrite() {
return nil, -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new layers at %q", r.layerspath())

7
pkg/config/config.go
View File

@ -122,6 +122,13 @@ type OptionsConfig struct {
// for shared image content
AdditionalImageStores []string `toml:"additionalimagestores"`
// AdditionalLayerStores is the location of additional read/only
// Layer stores. Usually used to access Networked File System
// for shared image content
// This API is experimental and can be changed without bumping the
// major version number.
AdditionalLayerStores []string `toml:"additionallayerstores"`
// Size
Size string `toml:"size"`

110
store.go
View File

@ -2,6 +2,7 @@ package storage
import (
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/ioutil"
@ -489,6 +490,30 @@ type Store interface {
// GetDigestLock returns digest-specific Locker.
GetDigestLock(digest.Digest) (Locker, error)
// LayerFromAdditionalLayerStore searches layers from the additional layer store and
// returns the object for handling this. Note that this hasn't been stored to this store
// yet so this needs to be done through PutAs method.
// Releasing AdditionalLayer handler is caller's responsibility.
// This API is experimental and can be changed without bumping the major version number.
LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error)
}
// AdditionalLayer reprents a layer that is contained in the additional layer store
// This API is experimental and can be changed without bumping the major version number.
type AdditionalLayer interface {
// PutAs creates layer based on this handler, using diff contents from the additional
// layer store.
PutAs(id, parent string, names []string) (*Layer, error)
// UncompressedDigest returns the uncompressed digest of this layer
UncompressedDigest() digest.Digest
// CompressedSize returns the compressed size of this layer
CompressedSize() int64
// Release tells the additional layer store that we don't use this handler.
Release()
}
type AutoUserNsOptions = types.AutoUserNsOptions
@ -3134,6 +3159,91 @@ func (s *store) Layer(id string) (*Layer, error) {
return nil, ErrLayerUnknown
}
func (s *store) LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error) {
adriver, ok := s.graphDriver.(drivers.AdditionalLayerStoreDriver)
if !ok {
return nil, ErrLayerUnknown
}
al, err := adriver.LookupAdditionalLayer(d, imageref)
if err != nil {
if errors.Is(err, drivers.ErrLayerUnknown) {
return nil, ErrLayerUnknown
}
return nil, err
}
info, err := al.Info()
if err != nil {
return nil, err
}
defer info.Close()
var layer Layer
if err := json.NewDecoder(info).Decode(&layer); err != nil {
return nil, err
}
return &additionalLayer{&layer, al, s}, nil
}
type additionalLayer struct {
layer *Layer
handler drivers.AdditionalLayer
s *store
}
func (al *additionalLayer) UncompressedDigest() digest.Digest {
return al.layer.UncompressedDigest
}
func (al *additionalLayer) CompressedSize() int64 {
return al.layer.CompressedSize
}
func (al *additionalLayer) PutAs(id, parent string, names []string) (*Layer, error) {
rlstore, err := al.s.LayerStore()
if err != nil {
return nil, err
}
rlstore.Lock()
defer rlstore.Unlock()
if modified, err := rlstore.Modified(); modified || err != nil {
if err = rlstore.Load(); err != nil {
return nil, err
}
}
rlstores, err := al.s.ROLayerStores()
if err != nil {
return nil, err
}
var parentLayer *Layer
if parent != "" {
for _, lstore := range append([]ROLayerStore{rlstore}, rlstores...) {
if lstore != rlstore {
lstore.RLock()
defer lstore.Unlock()
if modified, err := lstore.Modified(); modified || err != nil {
if err = lstore.Load(); err != nil {
return nil, err
}
}
}
parentLayer, err = lstore.Get(parent)
if err == nil {
break
}
}
if parentLayer == nil {
return nil, ErrLayerUnknown
}
}
return rlstore.PutAdditionalLayer(id, parentLayer, names, al.handler)
}
func (al *additionalLayer) Release() {
al.handler.Release()
}
func (s *store) Image(id string) (*Image, error) {
istore, err := s.ImageStore()
if err != nil {

3
types/options.go
View File

@ -300,6 +300,9 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) {
for _, s := range config.Storage.Options.AdditionalImageStores {
storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.imagestore=%s", config.Storage.Driver, s))
}
for _, s := range config.Storage.Options.AdditionalLayerStores {
storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.additionallayerstore=%s", config.Storage.Driver, s))
}
if config.Storage.Options.Size != "" {
storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.size=%s", config.Storage.Driver, config.Storage.Options.Size))
}