manifest-push: add support for --force-compression

Adds support for --force-compression which allows end-users to force
push blobs with the selected compresison in --compression option, in
order to make sure that blobs of other compression on registry are not
reused.

Signed-off-by: Aditya R <arajan@redhat.com>

cmd/podman/manifest/push.go

@@ -72,6 +72,8 @@ func init() {
 	flags.StringVar(&manifestPushOpts.DigestFile, digestfileFlagName, "", "after copying the image, write the digest of the resulting digest to the file")
 	_ = pushCmd.RegisterFlagCompletionFunc(digestfileFlagName, completion.AutocompleteDefault)
 
+	flags.BoolVar(&manifestPushOpts.ForceCompressionFormat, "force-compression", false, "Use the specified compression algorithm if the destination contains a differently-compressed variant already")
+
 	formatFlagName := "format"
 	flags.StringVarP(&manifestPushOpts.Format, formatFlagName, "f", "", "manifest type (oci or v2s2) to attempt to use when pushing the manifest list (default is manifest type of source)")
 	_ = pushCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteManifestFormat)

docs/source/markdown/podman-manifest-push.1.md.in

@@ -45,6 +45,8 @@ the list or index itself. (Default true)
 #### **--force-compression**
 
 Use the specified compression algorithm even if the destination contains a differently-compressed variant already.
+Usually use for this flag arises when image is prior compressed and pushed using `--compression-format` with a different
+compression algorithm and user now needs to overwrite those blobs with a new compression algorithm on the remote registry.
 
 #### **--format**, **-f**=*format*
 

docs/source/markdown/podman-push.1.md.in

@@ -70,9 +70,11 @@ Layer(s) to encrypt: 0-indexed layer indices with support for negative indexing
 
 The [protocol:keyfile] specifies the encryption protocol, which can be JWE (RFC7516), PGP (RFC4880), and PKCS7 (RFC2315) and the key material required for image encryption. For instance, jwe:/path/to/key.pem or pgp:admin@example.com or pkcs7:/path/to/x509-file.
 
-### **--force-compression**
+#### **--force-compression**
 
 Use the specified compression algorithm even if the destination contains a differently-compressed variant already.
+Usually use for this flag arises when image is prior compressed and pushed using `--compression-format` with a different
+compression algorithm and user now needs to overwrite those blobs with a new compression algorithm on the remote registry.
 
 #### **--format**, **-f**=*format*
 

pkg/api/handlers/libpod/manifests.go

@@ -333,14 +333,15 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 	decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)
 
 	query := struct {
-		All               bool     `schema:"all"`
-		CompressionFormat string   `schema:"compressionFormat"`
-		CompressionLevel  *int     `schema:"compressionLevel"`
-		Format            string   `schema:"format"`
-		RemoveSignatures  bool     `schema:"removeSignatures"`
-		TLSVerify         bool     `schema:"tlsVerify"`
-		Quiet             bool     `schema:"quiet"`
-		AddCompression    []string `schema:"addCompression"`
+		All                    bool     `schema:"all"`
+		CompressionFormat      string   `schema:"compressionFormat"`
+		CompressionLevel       *int     `schema:"compressionLevel"`
+		ForceCompressionFormat bool     `schema:"forceCompressionFormat"`
+		Format                 string   `schema:"format"`
+		RemoveSignatures       bool     `schema:"removeSignatures"`
+		TLSVerify              bool     `schema:"tlsVerify"`
+		Quiet                  bool     `schema:"quiet"`
+		AddCompression         []string `schema:"addCompression"`
 	}{
 		// Add defaults here once needed.
 		TLSVerify: true,
@@ -372,16 +373,17 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 		password = authconf.Password
 	}
 	options := entities.ImagePushOptions{
-		All:               query.All,
-		Authfile:          authfile,
-		AddCompression:    query.AddCompression,
-		CompressionFormat: query.CompressionFormat,
-		CompressionLevel:  query.CompressionLevel,
-		Format:            query.Format,
-		Password:          password,
-		Quiet:             true,
-		RemoveSignatures:  query.RemoveSignatures,
-		Username:          username,
+		All:                    query.All,
+		Authfile:               authfile,
+		AddCompression:         query.AddCompression,
+		CompressionFormat:      query.CompressionFormat,
+		CompressionLevel:       query.CompressionLevel,
+		ForceCompressionFormat: query.ForceCompressionFormat,
+		Format:                 query.Format,
+		Password:               password,
+		Quiet:                  true,
+		RemoveSignatures:       query.RemoveSignatures,
+		Username:               username,
 	}
 	if sys := runtime.SystemContext(); sys != nil {
 		options.CertDir = sys.DockerCertPath

pkg/api/server/register_manifest.go

@@ -67,6 +67,11 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error {
 	//    type: array
 	//    items:
 	//      type: string
+	//  - in: query
+	//    name: forceCompressionFormat
+	//    description: Use the specified compression algorithm if the destination contains a differently-compressed variant already.
+	//    type: boolean
+	//    default: false
 	//  - in: path
 	//    name: destination
 	//    type: string

pkg/domain/infra/abi/manifest.go

@@ -346,6 +346,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin
 	pushOptions.Writer = opts.Writer
 	pushOptions.CompressionLevel = opts.CompressionLevel
 	pushOptions.AddCompression = opts.AddCompression
+	pushOptions.ForceCompressionFormat = opts.ForceCompressionFormat
 
 	compressionFormat := opts.CompressionFormat
 	if compressionFormat == "" {

pkg/domain/infra/tunnel/manifest.go

@@ -135,7 +135,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin
 	}
 
 	options := new(images.PushOptions)
-	options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer).WithAddCompression(opts.AddCompression)
+	options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer).WithAddCompression(opts.AddCompression).WithForceCompressionFormat(opts.ForceCompressionFormat)
 
 	if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined {
 		if s == types.OptionalBoolTrue {

test/e2e/manifest_test.go

@@ -154,7 +154,7 @@ var _ = Describe("Podman manifest", func() {
 		Expect(session2.OutputToString()).To(Equal(session.OutputToString()))
 	})
 
-	It("push with --add-compression", func() {
+	It("push with --add-compression and --force-compression", func() {
 		if podmanTest.Host.Arch == "ppc64le" {
 			Skip("No registry image for ppc64le")
 		}
@@ -209,6 +209,49 @@ var _ = Describe("Podman manifest", func() {
 		Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue())
 		Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeTrue())
 		Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeTrue())
+
+		// Note: Pushing again with --force-compression should produce the correct response the since blobs will be correctly force-pushed again.
+		push = podmanTest.Podman([]string{"manifest", "push", "--all", "--add-compression", "zstd", "--tls-verify=false", "--compression-format", "gzip", "--force-compression", "--remove-signatures", "foobar", "localhost:5000/list"})
+		push.WaitWithDefaultTimeout()
+		Expect(push).Should(Exit(0))
+		output = push.ErrorToString()
+		// 4 images must be pushed two for gzip and two for zstd
+		Expect(output).To(ContainSubstring("Copying 4 images generated from 2 images in list"))
+
+		session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/list:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		inspectData = []byte(session.OutputToString())
+		err = json.Unmarshal(inspectData, &index)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(verifyInstanceCompression(index.Manifests, "zstd", "amd64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeTrue())
+
+		// Note: Pushing again without --force-compression should produce in-correct/wrong result since blobs are already present in registry so they will be reused
+		// ignoring our compression priority ( this is expected behaviour of c/image and --force-compression is introduced to mitigate this behaviour ).
+		push = podmanTest.Podman([]string{"manifest", "push", "--all", "--add-compression", "zstd", "--tls-verify=false", "--remove-signatures", "foobar", "localhost:5000/list"})
+		push.WaitWithDefaultTimeout()
+		Expect(push).Should(Exit(0))
+		output = push.ErrorToString()
+		// 4 images must be pushed two for gzip and two for zstd
+		Expect(output).To(ContainSubstring("Copying 4 images generated from 2 images in list"))
+
+		session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/list:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		inspectData = []byte(session.OutputToString())
+		err = json.Unmarshal(inspectData, &index)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(verifyInstanceCompression(index.Manifests, "zstd", "amd64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue())
+		// blobs of zstd will be wrongly reused for gzip instances without --force-compression
+		Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeFalse())
+		// blobs of zstd will be wrongly reused for gzip instances without --force-compression
+		Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeFalse())
 	})
 
 	It("add --all", func() {