podman save: add `--uncompressed`

Add an option to `podman save` to allow uncompressed layers when
copying OCI images.  Do the neccessary plumbing for the remote client,
add tests and vendor in the latest commit from c/common to fetch
the neccessary changes in libimage.

Closes: #11613
Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

cmd/podman/images/save.go

@@ -84,6 +84,8 @@ func saveFlags(cmd *cobra.Command) {
 
 	flags.BoolVar(&saveOpts.Compress, "compress", false, "Compress tarball image layers when saving to a directory using the 'dir' transport. (default is same compression type as source)")
 
+	flags.BoolVar(&saveOpts.OciAcceptUncompressedLayers, "uncompressed", false, "Accept uncompressed layers when copying OCI images")
+
 	formatFlagName := "format"
 	flags.StringVar(&saveOpts.Format, formatFlagName, define.V2s2Archive, "Save image to oci-archive, oci-dir (directory with oci manifest type), docker-archive, docker-dir (directory with v2s2 manifest type)")
 	_ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteImageSaveFormat)

docs/source/markdown/podman-save.1.md

@@ -29,6 +29,10 @@ Note: `:` is a restricted character and cannot be part of the file name.
 Compress tarball image layers when pushing to a directory using the 'dir' transport. (default is same compression type, compressed or uncompressed, as source)
 Note: This flag can only be set when using the **dir** transport i.e --format=oci-dir or --format=docker-dir
 
+#### **--uncompressed**
+
+Accept uncompressed layers when copying OCI images.
+
 #### **--output**, **-o**=*file*
 
 Write to a file, default is STDOUT

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/containernetworking/cni v0.8.1
 	github.com/containernetworking/plugins v0.9.1
 	github.com/containers/buildah v1.23.0
-	github.com/containers/common v0.44.1-0.20210920093543-bf187ada7d0e
+	github.com/containers/common v0.44.1-0.20210921143342-f2f10e650c73
 	github.com/containers/conmon v2.0.20+incompatible
 	github.com/containers/image/v5 v5.16.0
 	github.com/containers/ocicrypt v1.1.2

go.sum

@@ -246,8 +246,8 @@ github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRD
 github.com/containers/buildah v1.23.0 h1:qGIeSNOczUHzvnaaOS29HSMiYAjw6JgIXYksAyvqnLs=
 github.com/containers/buildah v1.23.0/go.mod h1:K0iMKgy/MffkkgELBXhSXwTy2HTT6hM0X8qruDR1FwU=
 github.com/containers/common v0.44.0/go.mod h1:7sdP4vmI5Bm6FPFxb3lvAh1Iktb6tiO1MzjUzhxdoGo=
-github.com/containers/common v0.44.1-0.20210920093543-bf187ada7d0e h1:p21+CJSeryr0Vb3dottjXRNYTaRND1QSPm36NogQ7cQ=
-github.com/containers/common v0.44.1-0.20210920093543-bf187ada7d0e/go.mod h1:zxv7KjdYddSGoWuLUVp6eSb++Ow1zmSMB2jwxuNB4cU=
+github.com/containers/common v0.44.1-0.20210921143342-f2f10e650c73 h1:+qKOyTHbuFo3GPsrUksphfHxYMIJQmPgwpDdQnARGAI=
+github.com/containers/common v0.44.1-0.20210921143342-f2f10e650c73/go.mod h1:zxv7KjdYddSGoWuLUVp6eSb++Ow1zmSMB2jwxuNB4cU=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
 github.com/containers/image/v5 v5.16.0 h1:WQcNSzb7+ngS2cfynx0vUwhk+scpgiKlldVcsF8GPbI=

pkg/api/handlers/libpod/images.go

@@ -289,9 +289,10 @@ func ExportImages(w http.ResponseWriter, r *http.Request) {
 	runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime)
 	decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)
 	query := struct {
-		Compress   bool     `schema:"compress"`
-		Format     string   `schema:"format"`
-		References []string `schema:"references"`
+		Compress                    bool     `schema:"compress"`
+		Format                      string   `schema:"format"`
+		OciAcceptUncompressedLayers bool     `schema:"ociAcceptUncompressedLayers"`
+		References                  []string `schema:"references"`
 	}{
 		Format: define.OCIArchive,
 	}
@@ -353,11 +354,12 @@ func ExportImages(w http.ResponseWriter, r *http.Request) {
 
 	// Use the ABI image engine to share as much code as possible.
 	opts := entities.ImageSaveOptions{
-		Compress:          query.Compress,
-		Format:            query.Format,
-		MultiImageArchive: len(query.References) > 1,
-		Output:            output,
-		RemoveSignatures:  true,
+		Compress:                    query.Compress,
+		Format:                      query.Format,
+		MultiImageArchive:           len(query.References) > 1,
+		OciAcceptUncompressedLayers: query.OciAcceptUncompressedLayers,
+		Output:                      output,
+		RemoveSignatures:            true,
 	}
 
 	imageEngine := abi.ImageEngine{Libpod: runtime}

pkg/api/server/register_images.go

@@ -1150,6 +1150,10 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error {
 	//    name: compress
 	//    type: boolean
 	//    description: use compression on image
+	//  - in: query
+	//    name: ociAcceptUncompressedLayers
+	//    type: boolean
+	//    description: accept uncompressed layers when copying OCI images
 	// produces:
 	// - application/json
 	// responses:

pkg/bindings/images/types.go

@@ -65,6 +65,8 @@ type ExportOptions struct {
 	Compress *bool
 	// Format of the output
 	Format *string
+	// Accept uncompressed layers when copying OCI images.
+	OciAcceptUncompressedLayers *bool
 }
 
 //go:generate go run ../generator/generator.go PruneOptions

pkg/bindings/images/types_export_options.go

@@ -46,3 +46,18 @@ func (o *ExportOptions) GetFormat() string {
 	}
 	return *o.Format
 }
+
+// WithOciAcceptUncompressedLayers set field OciAcceptUncompressedLayers to given value
+func (o *ExportOptions) WithOciAcceptUncompressedLayers(value bool) *ExportOptions {
+	o.OciAcceptUncompressedLayers = &value
+	return o
+}
+
+// GetOciAcceptUncompressedLayers returns value of field OciAcceptUncompressedLayers
+func (o *ExportOptions) GetOciAcceptUncompressedLayers() bool {
+	if o.OciAcceptUncompressedLayers == nil {
+		var z bool
+		return z
+	}
+	return *o.OciAcceptUncompressedLayers
+}

pkg/domain/entities/images.go

@@ -301,6 +301,8 @@ type ImageSaveOptions struct {
 	// than one image.  Additional tags will be interpreted as references
 	// to images which are added to the archive.
 	MultiImageArchive bool
+	// Accept uncompressed layers when copying OCI images.
+	OciAcceptUncompressedLayers bool
 	// Output - write image to the specified path.
 	Output string
 	// Do not save the signature from the source image

pkg/domain/infra/abi/images.go

@@ -367,6 +367,7 @@ func (ir *ImageEngine) Load(ctx context.Context, options entities.ImageLoadOptio
 func (ir *ImageEngine) Save(ctx context.Context, nameOrID string, tags []string, options entities.ImageSaveOptions) error {
 	saveOptions := &libimage.SaveOptions{}
 	saveOptions.DirForceCompress = options.Compress
+	saveOptions.OciAcceptUncompressedLayers = options.OciAcceptUncompressedLayers
 	saveOptions.RemoveSignatures = options.RemoveSignatures
 
 	if !options.Quiet {

pkg/domain/infra/tunnel/images.go

@@ -256,6 +256,7 @@ func (ir *ImageEngine) Save(ctx context.Context, nameOrID string, tags []string,
 		err error
 	)
 	options := new(images.ExportOptions).WithFormat(opts.Format).WithCompress(opts.Compress)
+	options = options.WithOciAcceptUncompressedLayers(opts.OciAcceptUncompressedLayers)
 
 	switch opts.Format {
 	case "oci-dir", "docker-dir":

test/system/120-load.bats

@@ -183,4 +183,16 @@ verify_iid_and_name() {
     run_podman rmi -f $img1 $img2
 }
 
+@test "podman save --oci-accept-uncompressed-layers" {
+    archive=$PODMAN_TMPDIR/myimage-$(random_string 8).tar
+    untar=$PODMAN_TMPDIR/myuntar-$(random_string 8)
+    mkdir -p $untar
+
+    # Create a tarball, unpack it and make sure the layers are uncompressed.
+    run_podman save -o $archive --format oci-archive --uncompressed $IMAGE
+    run tar -C $untar -xvf $archive
+    run file $untar/blobs/sha256/*
+    is "$output" ".*POSIX tar archive" "layers are uncompressed"
+}
+
 # vim: filetype=sh

vendor/github.com/containers/common/libimage/copier.go

@@ -12,6 +12,7 @@ import (
 	"github.com/containers/common/pkg/retry"
 	"github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/pkg/compression"
 	"github.com/containers/image/v5/signature"
 	storageTransport "github.com/containers/image/v5/storage"
 	"github.com/containers/image/v5/types"
@@ -40,6 +41,10 @@ type CopyOptions struct {
 	// Allows for customizing the destination reference lookup.  This can
 	// be used to use custom blob caches.
 	DestinationLookupReferenceFunc LookupReferenceFunc
+	// CompressionFormat is the format to use for the compression of the blobs
+	CompressionFormat *compression.Algorithm
+	// CompressionLevel specifies what compression level is used
+	CompressionLevel *int
 
 	// containers-auth.json(5) file to use when authenticating against
 	// container registries.
@@ -65,6 +70,8 @@ type CopyOptions struct {
 	// types.  Short forms (e.g., oci, v2s2) used by some tools are not
 	// supported.
 	ManifestMIMEType string
+	// Accept uncompressed layers when copying OCI images.
+	OciAcceptUncompressedLayers bool
 	// If OciEncryptConfig is non-nil, it indicates that an image should be
 	// encrypted.  The encryption options is derived from the construction
 	// of EncryptConfig object.  Note: During initial encryption process of
@@ -242,6 +249,17 @@ func (r *Runtime) newCopier(options *CopyOptions) (*copier, error) {
 		c.systemContext.DockerCertPath = options.CertDirPath
 	}
 
+	if options.CompressionFormat != nil {
+		c.systemContext.CompressionFormat = options.CompressionFormat
+	}
+
+	if options.CompressionLevel != nil {
+		c.systemContext.CompressionLevel = options.CompressionLevel
+	}
+
+	// NOTE: for the sake of consistency it's called Oci* in the CopyOptions.
+	c.systemContext.OCIAcceptUncompressedLayers = options.OciAcceptUncompressedLayers
+
 	policy, err := signature.DefaultPolicy(c.systemContext)
 	if err != nil {
 		return nil, err

vendor/github.com/containers/common/pkg/config/default.go

@@ -198,7 +198,6 @@ func DefaultConfig() (*Config, error) {
 			TZ:                 "",
 			Umask:              "0022",
 			UTSNS:              "private",
-			UserNS:             "host",
 			UserNSSize:         DefaultUserNSSize,
 		},
 		Network: NetworkConfig{

vendor/github.com/containers/common/pkg/secrets/secrets.go

@@ -24,8 +24,8 @@ const secretIDLength = 25
 // errInvalidPath indicates that the secrets path is invalid
 var errInvalidPath = errors.New("invalid secrets path")
 
-// errNoSuchSecret indicates that the secret does not exist
-var errNoSuchSecret = errors.New("no such secret")
+// ErrNoSuchSecret indicates that the secret does not exist
+var ErrNoSuchSecret = errors.New("no such secret")
 
 // errSecretNameInUse indicates that the secret name is already in use
 var errSecretNameInUse = errors.New("secret name in use")
@@ -152,7 +152,7 @@ func (s *SecretsManager) Store(name string, data []byte, driverType string, driv
 		newID = newID[0:secretIDLength]
 		_, err := s.lookupSecret(newID)
 		if err != nil {
-			if errors.Cause(err) == errNoSuchSecret {
+			if errors.Cause(err) == ErrNoSuchSecret {
 				secr.ID = newID
 				break
 			} else {

vendor/github.com/containers/common/pkg/secrets/secretsdb.go

@@ -71,14 +71,14 @@ func (s *SecretsManager) getNameAndID(nameOrID string) (name, id string, err err
 	name, id, err = s.getExactNameAndID(nameOrID)
 	if err == nil {
 		return name, id, nil
-	} else if errors.Cause(err) != errNoSuchSecret {
+	} else if errors.Cause(err) != ErrNoSuchSecret {
 		return "", "", err
 	}
 
 	// ID prefix may have been given, iterate through all IDs.
 	// ID and partial ID has a max length of 25, so we return if its greater than that.
 	if len(nameOrID) > secretIDLength {
-		return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
+		return "", "", errors.Wrapf(ErrNoSuchSecret, "no secret with name or id %q", nameOrID)
 	}
 	exists := false
 	var foundID, foundName string
@@ -96,7 +96,7 @@ func (s *SecretsManager) getNameAndID(nameOrID string) (name, id string, err err
 	if exists {
 		return foundName, foundID, nil
 	}
-	return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
+	return "", "", errors.Wrapf(ErrNoSuchSecret, "no secret with name or id %q", nameOrID)
 }
 
 // getExactNameAndID takes a secret's name or ID and returns both its name and full ID.
@@ -115,7 +115,7 @@ func (s *SecretsManager) getExactNameAndID(nameOrID string) (name, id string, er
 		return name, id, nil
 	}
 
-	return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
+	return "", "", errors.Wrapf(ErrNoSuchSecret, "no secret with name or id %q", nameOrID)
 }
 
 // exactSecretExists checks if the secret exists, given a name or ID
@@ -123,7 +123,7 @@ func (s *SecretsManager) getExactNameAndID(nameOrID string) (name, id string, er
 func (s *SecretsManager) exactSecretExists(nameOrID string) (bool, error) {
 	_, _, err := s.getExactNameAndID(nameOrID)
 	if err != nil {
-		if errors.Cause(err) == errNoSuchSecret {
+		if errors.Cause(err) == ErrNoSuchSecret {
 			return false, nil
 		}
 		return false, err
@@ -158,7 +158,7 @@ func (s *SecretsManager) lookupSecret(nameOrID string) (*Secret, error) {
 		return &secret, nil
 	}
 
-	return nil, errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
+	return nil, errors.Wrapf(ErrNoSuchSecret, "no secret with name or id %q", nameOrID)
 }
 
 // Store creates a new secret in the secrets database.

vendor/modules.txt

@@ -94,7 +94,7 @@ github.com/containers/buildah/pkg/rusage
 github.com/containers/buildah/pkg/sshagent
 github.com/containers/buildah/pkg/util
 github.com/containers/buildah/util
-# github.com/containers/common v0.44.1-0.20210920093543-bf187ada7d0e
+# github.com/containers/common v0.44.1-0.20210921143342-f2f10e650c73
 github.com/containers/common/libimage
 github.com/containers/common/libimage/manifests
 github.com/containers/common/pkg/apparmor