Support additional layer store

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

drivers/driver.go

@@ -14,6 +14,7 @@ import (
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/directory"
 	"github.com/containers/storage/pkg/idtools"
+	digest "github.com/opencontainers/go-digest"
 )
 
 // FsMagic unsigned id of the filesystem in use.
@@ -33,7 +34,9 @@ var (
 	// ErrPrerequisites returned when driver does not meet prerequisites.
 	ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)")
 	// ErrIncompatibleFS returned when file system is not supported.
-	ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver")
+	ErrIncompatibleFS = errors.New("backing file system is unsupported for this graph driver")
+	// ErrLayerUnknown returned when the specified layer is unknown by the driver.
+	ErrLayerUnknown = errors.New("unknown layer")
 )
 
 //CreateOpts contains optional arguments for Create() and CreateReadWrite()
@@ -117,6 +120,7 @@ type ProtoDriver interface {
 	// known to this driver.
 	Cleanup() error
 	// AdditionalImageStores returns additional image stores supported by the driver
+	// This API is experimental and can be changed without bumping the major version number.
 	AdditionalImageStores() []string
 }
 
@@ -180,6 +184,30 @@ type CapabilityDriver interface {
 	Capabilities() Capabilities
 }
 
+// AdditionalLayer reprents a layer that is stored in the additional layer store
+// This API is experimental and can be changed without bumping the major version number.
+type AdditionalLayer interface {
+	// CreateAs creates a new layer from this additional layer
+	CreateAs(id, parent string) error
+
+	// Info returns arbitrary information stored along with this layer (i.e. `info` file)
+	Info() (io.ReadCloser, error)
+
+	// Release tells the additional layer store that we don't use this handler.
+	Release()
+}
+
+// AdditionalLayerStoreDriver is the interface for driver that supports
+// additional layer store functionality.
+// This API is experimental and can be changed without bumping the major version number.
+type AdditionalLayerStoreDriver interface {
+	Driver
+
+	// LookupAdditionalLayer looks up additional layer store by the specified
+	// digest and ref and returns an object representing that layer.
+	LookupAdditionalLayer(d digest.Digest, ref string) (AdditionalLayer, error)
+}
+
 // DiffGetterDriver is the interface for layered file system drivers that
 // provide a specialized function for getting file contents for tar-split.
 type DiffGetterDriver interface {

drivers/overlay/overlay.go

@@ -4,6 +4,7 @@ package overlay
 
 import (
 	"bytes"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -31,6 +32,7 @@ import (
 	"github.com/containers/storage/pkg/unshare"
 	units "github.com/docker/go-units"
 	"github.com/hashicorp/go-multierror"
+	digest "github.com/opencontainers/go-digest"
 	rsystem "github.com/opencontainers/runc/libcontainer/system"
 	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/opencontainers/selinux/go-selinux/label"
@@ -94,6 +96,7 @@ const (
 
 type overlayOptions struct {
 	imageStores       []string
+	layerStores       []additionalLayerStore
 	quota             quota.Quota
 	mountProgram      string
 	skipMountHome     bool
@@ -119,6 +122,17 @@ type Driver struct {
 	locker           *locker.Locker
 }
 
+type additionalLayerStore struct {
+
+	// path is the directory where this store is available on the host.
+	path string
+
+	// withReference is true when the store contains image reference information (base64-encoded)
+	// in its layer search path so the path to the diff will be
+	//  <path>/base64(reference)/<layerdigest>/
+	withReference bool
+}
+
 var (
 	backingFs             = "<unknown>"
 	projectQuotaSupported = false
@@ -397,6 +411,42 @@ func parseOptions(options []string) (*overlayOptions, error) {
 				}
 				o.imageStores = append(o.imageStores, store)
 			}
+		case "additionallayerstore":
+			logrus.Debugf("overlay: additionallayerstore=%s", val)
+			// Additional read only layer stores to use for lower paths
+			if val == "" {
+				continue
+			}
+			for _, lstore := range strings.Split(val, ",") {
+				elems := strings.Split(lstore, ":")
+				lstore = filepath.Clean(elems[0])
+				if !filepath.IsAbs(lstore) {
+					return nil, fmt.Errorf("overlay: additionallayerstore path %q is not absolute.  Can not be relative", lstore)
+				}
+				st, err := os.Stat(lstore)
+				if err != nil {
+					return nil, errors.Wrap(err, "overlay: can't stat additionallayerstore dir")
+				}
+				if !st.IsDir() {
+					return nil, fmt.Errorf("overlay: additionallayerstore path %q must be a directory", lstore)
+				}
+				var withReference bool
+				for _, e := range elems[1:] {
+					switch e {
+					case "ref":
+						if withReference {
+							return nil, fmt.Errorf("overlay: additionallayerstore config of %q contains %q option twice", lstore, e)
+						}
+						withReference = true
+					default:
+						return nil, fmt.Errorf("overlay: additionallayerstore config %q contains unknown option %q", lstore, e)
+					}
+				}
+				o.layerStores = append(o.layerStores, additionalLayerStore{
+					path:          lstore,
+					withReference: withReference,
+				})
+			}
 		case "mount_program":
 			logrus.Debugf("overlay: mount_program=%s", val)
 			if val != "" {
@@ -657,6 +707,24 @@ func (d *Driver) Cleanup() error {
 	return mount.Unmount(d.home)
 }
 
+// LookupAdditionalLayer looks up additional layer store by the specified
+// digest and ref and returns an object representing that layer.
+// This API is experimental and can be changed without bumping the major version number.
+func (d *Driver) LookupAdditionalLayer(dgst digest.Digest, ref string) (graphdriver.AdditionalLayer, error) {
+	l, err := d.getAdditionalLayerPath(dgst, ref)
+	if err != nil {
+		return nil, err
+	}
+	// Tell the additional layer store that we use this layer.
+	// This will increase reference counter on the store's side.
+	// This will be decreased on Release() method.
+	notifyUseAdditionalLayer(l)
+	return &additionalLayer{
+		path: l,
+		d:    d,
+	}, nil
+}
+
 // CreateFromTemplate creates a layer with the same contents and parent as another layer.
 func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error {
 	if readWrite {
@@ -959,6 +1027,8 @@ func (d *Driver) Remove(id string) error {
 		}
 	}
 
+	d.releaseAdditionalLayerByID(id)
+
 	if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) {
 		return err
 	}
@@ -1422,7 +1492,10 @@ func (f fileGetNilCloser) Close() error {
 // DiffGetter returns a FileGetCloser that can read files from the directory that
 // contains files for the layer differences. Used for direct access for tar-split.
 func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) {
-	p := d.getDiffPath(id)
+	p, err := d.getDiffPath(id)
+	if err != nil {
+		return nil, err
+	}
 	return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil
 }
 
@@ -1444,7 +1517,10 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts)
 		idMappings = &idtools.IDMappings{}
 	}
 
-	applyDir := d.getDiffPath(id)
+	applyDir, err := d.getDiffPath(id)
+	if err != nil {
+		return 0, err
+	}
 
 	logrus.Debugf("Applying tar in %s", applyDir)
 	// Overlay doesn't need the parent id to apply the diff
@@ -1462,10 +1538,23 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts)
 	return directory.Size(applyDir)
 }
 
-func (d *Driver) getDiffPath(id string) string {
+func (d *Driver) getDiffPath(id string) (string, error) {
 	dir := d.dir(id)
+	return redirectDiffIfAdditionalLayer(path.Join(dir, "diff"))
+}
 
-	return path.Join(dir, "diff")
+func (d *Driver) getLowerDiffPaths(id string) ([]string, error) {
+	layers, err := d.getLowerDirs(id)
+	if err != nil {
+		return nil, err
+	}
+	for i, l := range layers {
+		layers[i], err = redirectDiffIfAdditionalLayer(l)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return layers, nil
 }
 
 // DiffSize calculates the changes between the specified id
@@ -1476,7 +1565,11 @@ func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent stri
 		return d.naiveDiff.DiffSize(id, idMappings, parent, parentMappings, mountLabel)
 	}
 
-	return directory.Size(d.getDiffPath(id))
+	p, err := d.getDiffPath(id)
+	if err != nil {
+		return 0, err
+	}
+	return directory.Size(p)
 }
 
 // Diff produces an archive of the changes between the specified
@@ -1490,12 +1583,15 @@ func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string,
 		idMappings = &idtools.IDMappings{}
 	}
 
-	lowerDirs, err := d.getLowerDirs(id)
+	lowerDirs, err := d.getLowerDiffPaths(id)
 	if err != nil {
 		return nil, err
 	}
 
-	diffPath := d.getDiffPath(id)
+	diffPath, err := d.getDiffPath(id)
+	if err != nil {
+		return nil, err
+	}
 	logrus.Debugf("Tar with options on %s", diffPath)
 	return archive.TarWithOptions(diffPath, &archive.TarOptions{
 		Compression:    archive.Uncompressed,
@@ -1514,8 +1610,11 @@ func (d *Driver) Changes(id string, idMappings *idtools.IDMappings, parent strin
 	}
 	// Overlay doesn't have snapshots, so we need to get changes from all parent
 	// layers.
-	diffPath := d.getDiffPath(id)
+	diffPath, err := d.getDiffPath(id)
-	layers, err := d.getLowerDirs(id)
+	if err != nil {
+		return nil, err
+	}
+	layers, err := d.getLowerDiffPaths(id)
 	if err != nil {
 		return nil, err
 	}
@@ -1627,3 +1726,139 @@ func nameWithSuffix(name string, number int) string {
 	}
 	return fmt.Sprintf("%s%d", name, number)
 }
+
+func (d *Driver) getAdditionalLayerPath(dgst digest.Digest, ref string) (string, error) {
+	refElem := base64.StdEncoding.EncodeToString([]byte(ref))
+	for _, ls := range d.options.layerStores {
+		ref := ""
+		if ls.withReference {
+			ref = refElem
+		}
+		target := path.Join(ls.path, ref, dgst.String())
+		// Check if all necessary files exist
+		for _, p := range []string{
+			filepath.Join(target, "diff"),
+			filepath.Join(target, "info"),
+			// TODO(ktock): We should have an API to expose the stream data of this layer
+			//              to enable the client to retrieve the entire contents of this
+			//              layer when it exports this layer.
+		} {
+			if _, err := os.Stat(p); err != nil {
+				return "", errors.Wrapf(graphdriver.ErrLayerUnknown,
+					"failed to stat additional layer %q: %v", p, err)
+			}
+		}
+		return target, nil
+	}
+
+	return "", errors.Wrapf(graphdriver.ErrLayerUnknown,
+		"additional layer (%q, %q) not found", dgst, ref)
+}
+
+func (d *Driver) releaseAdditionalLayerByID(id string) {
+	if al, err := ioutil.ReadFile(path.Join(d.dir(id), "additionallayer")); err == nil {
+		notifyReleaseAdditionalLayer(string(al))
+	} else if !os.IsNotExist(err) {
+		logrus.Warnf("unexpected error on reading Additional Layer Store pointer %v", err)
+	}
+}
+
+// additionalLayer represents a layer in Additional Layer Store.
+type additionalLayer struct {
+	path        string
+	d           *Driver
+	releaseOnce sync.Once
+}
+
+// Info returns arbitrary information stored along with this layer (i.e. `info` file).
+// This API is experimental and can be changed without bumping the major version number.
+func (al *additionalLayer) Info() (io.ReadCloser, error) {
+	return os.Open(filepath.Join(al.path, "info"))
+}
+
+// CreateAs creates a new layer from this additional layer.
+// This API is experimental and can be changed without bumping the major version number.
+func (al *additionalLayer) CreateAs(id, parent string) error {
+	// TODO: support opts
+	if err := al.d.Create(id, parent, nil); err != nil {
+		return err
+	}
+	dir := al.d.dir(id)
+	diffDir := path.Join(dir, "diff")
+	if err := os.RemoveAll(diffDir); err != nil {
+		return err
+	}
+	// tell the additional layer store that we use this layer.
+	// mark this layer as "additional layer"
+	if err := ioutil.WriteFile(path.Join(dir, "additionallayer"), []byte(al.path), 0644); err != nil {
+		return err
+	}
+	notifyUseAdditionalLayer(al.path)
+	return os.Symlink(filepath.Join(al.path, "diff"), diffDir)
+}
+
+// Release tells the additional layer store that we don't use this handler.
+// This API is experimental and can be changed without bumping the major version number.
+func (al *additionalLayer) Release() {
+	// Tell the additional layer store that we don't use this layer handler.
+	// This will decrease the reference counter on the store's side, which was
+	// increased in LookupAdditionalLayer (so this must be called only once).
+	al.releaseOnce.Do(func() {
+		notifyReleaseAdditionalLayer(al.path)
+	})
+}
+
+// notifyUseAdditionalLayer notifies Additional Layer Store that we use the specified layer.
+// This is done by creating "use" file in the layer directory. This is useful for
+// Additional Layer Store to consider when to perform GC. Notification-aware Additional
+// Layer Store must return ENOENT.
+func notifyUseAdditionalLayer(al string) {
+	if !path.IsAbs(al) {
+		logrus.Warnf("additionallayer must be absolute (got: %v)", al)
+		return
+	}
+	useFile := path.Join(al, "use")
+	f, err := os.Create(useFile)
+	if os.IsNotExist(err) {
+		return
+	} else if err == nil {
+		f.Close()
+		if err := os.Remove(useFile); err != nil {
+			logrus.Warnf("failed to remove use file")
+		}
+	}
+	logrus.Warnf("unexpected error by Additional Layer Store %v during use; GC doesn't seem to be supported", err)
+}
+
+// notifyReleaseAdditionalLayer notifies Additional Layer Store that we don't use the specified
+// layer anymore. This is done by rmdir-ing the layer directory. This is useful for
+// Additional Layer Store to consider when to perform GC. Notification-aware Additional
+// Layer Store must return ENOENT.
+func notifyReleaseAdditionalLayer(al string) {
+	if !path.IsAbs(al) {
+		logrus.Warnf("additionallayer must be absolute (got: %v)", al)
+		return
+	}
+	// tell the additional layer store that we don't use this layer anymore.
+	err := unix.Rmdir(al)
+	if os.IsNotExist(err) {
+		return
+	}
+	logrus.Warnf("unexpected error by Additional Layer Store %v during release; GC doesn't seem to be supported", err)
+}
+
+// redirectDiffIfAdditionalLayer checks if the passed diff path is Additional Layer and
+// returns the redirected path. If the passed diff is not the one in Additional Layer
+// Store, it returns the original path without changes.
+func redirectDiffIfAdditionalLayer(diffPath string) (string, error) {
+	if ld, err := os.Readlink(diffPath); err == nil {
+		// diff is the link to Additional Layer Store
+		if !path.IsAbs(ld) {
+			return "", fmt.Errorf("linkpath must be absolute (got: %q)", ld)
+		}
+		diffPath = ld
+	} else if err.(*os.PathError).Err != syscall.EINVAL {
+		return "", err
+	}
+	return diffPath, nil
+}

layers.go

@@ -250,6 +250,11 @@ type LayerStore interface {
 	// LoadLocked wraps Load in a locked state. This means it loads the store
 	// and cleans-up invalid layers if needed.
 	LoadLocked() error
+
+	// PutAdditionalLayer creates a layer using the diff contained in the additional layer
+	// store.
+	// This API is experimental and can be changed without bumping the major version number.
+	PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error)
 }
 
 type layerStore struct {
@@ -610,6 +615,58 @@ func (r *layerStore) Status() ([][2]string, error) {
 	return r.driver.Status(), nil
 }
 
+func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) {
+	if duplicateLayer, idInUse := r.byid[id]; idInUse {
+		return duplicateLayer, ErrDuplicateID
+	}
+	for _, name := range names {
+		if _, nameInUse := r.byname[name]; nameInUse {
+			return nil, ErrDuplicateName
+		}
+	}
+
+	parent := ""
+	if parentLayer != nil {
+		parent = parentLayer.ID
+	}
+
+	info, err := aLayer.Info()
+	if err != nil {
+		return nil, err
+	}
+	defer info.Close()
+	layer = &Layer{}
+	if err := json.NewDecoder(info).Decode(layer); err != nil {
+		return nil, err
+	}
+	layer.ID = id
+	layer.Parent = parent
+	layer.Created = time.Now().UTC()
+
+	if err := aLayer.CreateAs(id, parent); err != nil {
+		return nil, err
+	}
+
+	// TODO: check if necessary fields are filled
+	r.layers = append(r.layers, layer)
+	r.idindex.Add(id)
+	r.byid[id] = layer
+	for _, name := range names { // names got from the additional layer store won't be used
+		r.byname[name] = layer
+	}
+	if layer.CompressedDigest != "" {
+		r.bycompressedsum[layer.CompressedDigest] = append(r.bycompressedsum[layer.CompressedDigest], layer.ID)
+	}
+	if layer.UncompressedDigest != "" {
+		r.byuncompressedsum[layer.CompressedDigest] = append(r.byuncompressedsum[layer.CompressedDigest], layer.ID)
+	}
+	if err := r.Save(); err != nil {
+		r.driver.Remove(id)
+		return nil, err
+	}
+	return copyLayer(layer), nil
+}
+
 func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (layer *Layer, size int64, err error) {
 	if !r.IsReadWrite() {
 		return nil, -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new layers at %q", r.layerspath())

pkg/config/config.go

@@ -122,6 +122,13 @@ type OptionsConfig struct {
 	// for shared image content
 	AdditionalImageStores []string `toml:"additionalimagestores"`
 
+	// AdditionalLayerStores is the location of additional read/only
+	// Layer stores.  Usually used to access Networked File System
+	// for shared image content
+	// This API is experimental and can be changed without bumping the
+	// major version number.
+	AdditionalLayerStores []string `toml:"additionallayerstores"`
+
 	// Size
 	Size string `toml:"size"`
 

store.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -489,6 +490,30 @@ type Store interface {
 
 	// GetDigestLock returns digest-specific Locker.
 	GetDigestLock(digest.Digest) (Locker, error)
+
+	// LayerFromAdditionalLayerStore searches layers from the additional layer store and
+	// returns the object for handling this. Note that this hasn't been stored to this store
+	// yet so this needs to be done through PutAs method.
+	// Releasing AdditionalLayer handler is caller's responsibility.
+	// This API is experimental and can be changed without bumping the major version number.
+	LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error)
+}
+
+// AdditionalLayer reprents a layer that is contained in the additional layer store
+// This API is experimental and can be changed without bumping the major version number.
+type AdditionalLayer interface {
+	// PutAs creates layer based on this handler, using diff contents from the additional
+	// layer store.
+	PutAs(id, parent string, names []string) (*Layer, error)
+
+	// UncompressedDigest returns the uncompressed digest of this layer
+	UncompressedDigest() digest.Digest
+
+	// CompressedSize returns the compressed size of this layer
+	CompressedSize() int64
+
+	// Release tells the additional layer store that we don't use this handler.
+	Release()
 }
 
 type AutoUserNsOptions = types.AutoUserNsOptions
@@ -3134,6 +3159,91 @@ func (s *store) Layer(id string) (*Layer, error) {
 	return nil, ErrLayerUnknown
 }
 
+func (s *store) LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error) {
+	adriver, ok := s.graphDriver.(drivers.AdditionalLayerStoreDriver)
+	if !ok {
+		return nil, ErrLayerUnknown
+	}
+
+	al, err := adriver.LookupAdditionalLayer(d, imageref)
+	if err != nil {
+		if errors.Is(err, drivers.ErrLayerUnknown) {
+			return nil, ErrLayerUnknown
+		}
+		return nil, err
+	}
+	info, err := al.Info()
+	if err != nil {
+		return nil, err
+	}
+	defer info.Close()
+	var layer Layer
+	if err := json.NewDecoder(info).Decode(&layer); err != nil {
+		return nil, err
+	}
+	return &additionalLayer{&layer, al, s}, nil
+}
+
+type additionalLayer struct {
+	layer   *Layer
+	handler drivers.AdditionalLayer
+	s       *store
+}
+
+func (al *additionalLayer) UncompressedDigest() digest.Digest {
+	return al.layer.UncompressedDigest
+}
+
+func (al *additionalLayer) CompressedSize() int64 {
+	return al.layer.CompressedSize
+}
+
+func (al *additionalLayer) PutAs(id, parent string, names []string) (*Layer, error) {
+	rlstore, err := al.s.LayerStore()
+	if err != nil {
+		return nil, err
+	}
+	rlstore.Lock()
+	defer rlstore.Unlock()
+	if modified, err := rlstore.Modified(); modified || err != nil {
+		if err = rlstore.Load(); err != nil {
+			return nil, err
+		}
+	}
+	rlstores, err := al.s.ROLayerStores()
+	if err != nil {
+		return nil, err
+	}
+
+	var parentLayer *Layer
+	if parent != "" {
+		for _, lstore := range append([]ROLayerStore{rlstore}, rlstores...) {
+			if lstore != rlstore {
+				lstore.RLock()
+				defer lstore.Unlock()
+				if modified, err := lstore.Modified(); modified || err != nil {
+					if err = lstore.Load(); err != nil {
+						return nil, err
+					}
+				}
+			}
+			parentLayer, err = lstore.Get(parent)
+			if err == nil {
+				break
+			}
+		}
+		if parentLayer == nil {
+			return nil, ErrLayerUnknown
+		}
+	}
+
+	return rlstore.PutAdditionalLayer(id, parentLayer, names, al.handler)
+}
+
+func (al *additionalLayer) Release() {
+	al.handler.Release()
+}
+
 func (s *store) Image(id string) (*Image, error) {
 	istore, err := s.ImageStore()
 	if err != nil {

types/options.go

@@ -300,6 +300,9 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) {
 	for _, s := range config.Storage.Options.AdditionalImageStores {
 		storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.imagestore=%s", config.Storage.Driver, s))
 	}
+	for _, s := range config.Storage.Options.AdditionalLayerStores {
+		storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.additionallayerstore=%s", config.Storage.Driver, s))
+	}
 	if config.Storage.Options.Size != "" {
 		storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.size=%s", config.Storage.Driver, config.Storage.Options.Size))
 	}