store: new API ApplyStagedLayer

Add a race-condition-free alternative to using CreateLayer and
ApplyDiffFromStagingDirectory, ensuring the store is locked for the
entire duration while the layer is being created and populated.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

layers.go

@@ -181,6 +181,13 @@ type DiffOptions struct {
 	Compression *archive.Compression
 }
 
+// stagedLayerOptions are the options passed to .create to populate a staged
+// layer
+type stagedLayerOptions struct {
+	DiffOutput  *drivers.DriverWithDifferOutput
+	DiffOptions *drivers.ApplyDiffWithDifferOpts
+}
+
 // roLayerStore wraps a graph driver, adding the ability to refer to layers by
 // name, and keeping track of parent-child relationships, along with a list of
 // all known layers.
@@ -267,7 +274,7 @@ type rwLayerStore interface {
 	// underlying drivers do not themselves distinguish between writeable
 	// and read-only layers.  Returns the new layer structure and the size of the
 	// diff which was applied to its parent to initialize its contents.
-	create(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, diff io.Reader) (*Layer, int64, error)
+	create(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, diff io.Reader, slo *stagedLayerOptions) (*Layer, int64, error)
 
 	// updateNames modifies names associated with a layer based on (op, names).
 	updateNames(id string, names []string, op updateNameOperation) error
@@ -1232,7 +1239,7 @@ func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []s
 }
 
 // Requires startWriting.
-func (r *layerStore) create(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, diff io.Reader) (layer *Layer, size int64, err error) {
+func (r *layerStore) create(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, diff io.Reader, slo *stagedLayerOptions) (layer *Layer, size int64, err error) {
 	if moreOptions == nil {
 		moreOptions = &LayerOptions{}
 	}
@@ -1426,6 +1433,11 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount
 			cleanupFailureContext = "applying layer diff"
 			return nil, -1, err
 		}
+	} else if slo != nil {
+		if err := r.applyDiffFromStagingDirectory(layer.ID, slo.DiffOutput, slo.DiffOptions); err != nil {
+			cleanupFailureContext = "applying staged directory diff"
+			return nil, -1, err
+		}
 	} else {
 		// applyDiffWithOptions() would have updated r.bycompressedsum
 		// and r.byuncompressedsum for us, but if we used a template

store.go

@@ -71,6 +71,19 @@ type metadataStore interface {
 	rwMetadataStore
 }
 
+// ApplyStagedLayerOptions contains options to pass to ApplyStagedLayer
+type ApplyStagedLayerOptions struct {
+	ID           string        // Mandatory
+	ParentLayer  string        // Optional
+	Names        []string      // Optional
+	MountLabel   string        // Optional
+	Writeable    bool          // Optional
+	LayerOptions *LayerOptions // Optional
+
+	DiffOutput  *drivers.DriverWithDifferOutput  // Mandatory
+	DiffOptions *drivers.ApplyDiffWithDifferOpts // Mandatory
+}
+
 // An roBigDataStore wraps up the read-only big-data related methods of the
 // various types of file-based lookaside stores that we implement.
 type roBigDataStore interface {
@@ -318,12 +331,17 @@ type Store interface {
 	ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffWithDifferOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error)
 
 	// ApplyDiffFromStagingDirectory uses stagingDirectory to create the diff.
-	// Deprecated: it will be removed soon.
+	// Deprecated: it will be removed soon.  Use ApplyStagedLayer instead.
 	ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error
 
 	// CleanupStagingDirectory cleanups the staging directory.  It can be used to cleanup the staging directory on errors
 	CleanupStagingDirectory(stagingDirectory string) error
 
+	// ApplyStagedLayer combines the functions of CreateLayer and ApplyDiffFromStagingDirectory,
+	// marking the layer for automatic removal if applying the diff fails
+	// for any reason.
+	ApplyStagedLayer(args ApplyStagedLayerOptions) (*Layer, error)
+
 	// DifferTarget gets the path to the differ target.
 	DifferTarget(id string) (string, error)
 
@@ -1510,7 +1528,7 @@ func (s *store) putLayer(id, parent string, names []string, mountLabel string, w
 			GIDMap:         copyIDMap(gidMap),
 		}
 	}
-	return rlstore.create(id, parentLayer, names, mountLabel, nil, &layerOptions, writeable, diff)
+	return rlstore.create(id, parentLayer, names, mountLabel, nil, &layerOptions, writeable, diff, slo)
 }
 
 func (s *store) PutLayer(id, parent string, names []string, mountLabel string, writeable bool, lOptions *LayerOptions, diff io.Reader) (*Layer, int64, error) {
@@ -1724,7 +1742,7 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore roImageStore, rlst
 		}
 	}
 	layerOptions.TemplateLayer = layer.ID
-	mappedLayer, _, err := rlstore.create("", parentLayer, nil, layer.MountLabel, nil, &layerOptions, false, nil)
+	mappedLayer, _, err := rlstore.create("", parentLayer, nil, layer.MountLabel, nil, &layerOptions, false, nil, nil)
 	if err != nil {
 		return nil, fmt.Errorf("creating an ID-mapped copy of layer %q: %w", layer.ID, err)
 	}
@@ -1895,7 +1913,7 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat
 		options.Flags[mountLabelFlag] = mountLabel
 	}
 
-	clayer, _, err := rlstore.create(layer, imageTopLayer, nil, mlabel, options.StorageOpt, layerOptions, true, nil)
+	clayer, _, err := rlstore.create(layer, imageTopLayer, nil, mlabel, options.StorageOpt, layerOptions, true, nil, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -2972,6 +2990,16 @@ func (s *store) ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffO
 	return err
 }
 
+func (s *store) ApplyStagedLayer(args ApplyStagedLayerOptions) (*Layer, error) {
+	slo := stagedLayerOptions{
+		DiffOutput:  args.DiffOutput,
+		DiffOptions: args.DiffOptions,
+	}
+
+	layer, _, err := s.putLayer(args.ID, args.ParentLayer, args.Names, args.MountLabel, args.Writeable, args.LayerOptions, nil, &slo)
+	return layer, err
+}
+
 func (s *store) CleanupStagingDirectory(stagingDirectory string) error {
 	_, err := writeToLayerStore(s, func(rlstore rwLayerStore) (struct{}, error) {
 		return struct{}{}, rlstore.CleanupStagingDirectory(stagingDirectory)

userns.go

@@ -175,7 +175,7 @@ outer:
 
 	// We need to create a temporary layer so we can mount it and lookup the
 	// maximum IDs used.
-	clayer, _, err := rlstore.create("", topLayer, nil, "", nil, layerOptions, false, nil)
+	clayer, _, err := rlstore.create("", topLayer, nil, "", nil, layerOptions, false, nil, nil)
 	if err != nil {
 		return 0, err
 	}