Merge pull request #15125 from Romain-Geissler-1A/fix-manifest-push

[Closes 15109] Add flag "--compression-format" to "podman manifest push" (and other fixes)

cmd/podman/manifest/push.go

@@ -76,6 +76,10 @@ func init() {
 	flags.BoolVarP(&manifestPushOpts.Quiet, "quiet", "q", false, "don't output progress information when pushing lists")
 	flags.SetNormalizeFunc(utils.AliasFlags)
 
+	compressionFormat := "compression-format"
+	flags.StringVar(&manifestPushOpts.CompressionFormat, compressionFormat, "", "compression format to use")
+	_ = pushCmd.RegisterFlagCompletionFunc(compressionFormat, common.AutocompleteCompressionFormat)
+
 	if registry.IsRemote() {
 		_ = flags.MarkHidden("cert-dir")
 	}

docs/source/markdown/podman-manifest-push.1.md

@@ -32,6 +32,10 @@ environment variable. `export REGISTRY_AUTH_FILE=path`
 Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry. (Default: /etc/containers/certs.d)
 Please refer to containers-certs.d(5) for details. (This option is not available with the remote Podman client, including Mac and Windows (excluding WSL2) machines)
 
+#### **--compression-format**=**gzip** | *zstd* | *zstd:chunked*
+
+Specifies the compression format to use.  Supported values are: `gzip`, `zstd` and `zstd:chunked`.  The default is `gzip` unless overridden in the containers.conf file.
+
 #### **--creds**=*creds*
 
 The [username[:password]] to use to authenticate with the registry if required.

docs/source/markdown/podman-push.1.md

@@ -67,7 +67,7 @@ Note: This flag can only be set when using the **dir** transport
 
 #### **--compression-format**=**gzip** | *zstd* | *zstd:chunked*
 
-Specifies the compression format to use.  Supported values are: `gzip`, `zstd` and `zstd:chunked`.  The default is `gzip`.
+Specifies the compression format to use.  Supported values are: `gzip`, `zstd` and `zstd:chunked`.  The default is `gzip` unless overridden in the containers.conf file.
 
 #### **--creds**=*[username[:password]]*
 

pkg/api/handlers/libpod/images_push.go

@@ -26,6 +26,7 @@ func PushImage(w http.ResponseWriter, r *http.Request) {
 
 	query := struct {
 		All               bool   `schema:"all"`
+		CompressionFormat string `schema:"compressionFormat"`
 		Destination       string `schema:"destination"`
 		Format            string `schema:"format"`
 		RemoveSignatures  bool   `schema:"removeSignatures"`
@@ -73,6 +74,7 @@ func PushImage(w http.ResponseWriter, r *http.Request) {
 	options := entities.ImagePushOptions{
 		All:               query.All,
 		Authfile:          authfile,
+		CompressionFormat: query.CompressionFormat,
 		Format:            query.Format,
 		Password:          password,
 		Quiet:             true,

pkg/api/handlers/libpod/manifests.go

@@ -307,6 +307,9 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 
 	query := struct {
 		All               bool   `schema:"all"`
+		CompressionFormat string `schema:"compressionFormat"`
+		Format            string `schema:"format"`
+		RemoveSignatures  bool   `schema:"removeSignatures"`
 		TLSVerify         bool   `schema:"tlsVerify"`
 	}{
 		// Add defaults here once needed.
@@ -336,10 +339,13 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 		password = authconf.Password
 	}
 	options := entities.ImagePushOptions{
-		Authfile: authfile,
-		Username: username,
-		Password: password,
 		All:               query.All,
+		Authfile:          authfile,
+		CompressionFormat: query.CompressionFormat,
+		Format:            query.Format,
+		Password:          password,
+		RemoveSignatures:  query.RemoveSignatures,
+		Username:          username,
 	}
 	if sys := runtime.SystemContext(); sys != nil {
 		options.CertDir = sys.DockerCertPath

pkg/bindings/images/types.go

@@ -123,6 +123,8 @@ type PushOptions struct {
 	Authfile *string
 	// Compress tarball image layers when pushing to a directory using the 'dir' transport.
 	Compress *bool
+	// CompressionFormat is the format to use for the compression of the blobs
+	CompressionFormat *string
 	// Manifest type of the pushed image
 	Format *string
 	// Password for authenticating against the registry.

pkg/bindings/images/types_push_options.go

@@ -62,6 +62,21 @@ func (o *PushOptions) GetCompress() bool {
 	return *o.Compress
 }
 
+// WithCompressionFormat set field CompressionFormat to given value
+func (o *PushOptions) WithCompressionFormat(value string) *PushOptions {
+	o.CompressionFormat = &value
+	return o
+}
+
+// GetCompressionFormat returns value of field CompressionFormat
+func (o *PushOptions) GetCompressionFormat() string {
+	if o.CompressionFormat == nil {
+		var z string
+		return z
+	}
+	return *o.CompressionFormat
+}
+
 // WithFormat set field Format to given value
 func (o *PushOptions) WithFormat(value string) *PushOptions {
 	o.Format = &value

pkg/domain/infra/abi/manifest.go

@@ -13,6 +13,7 @@ import (
 	"github.com/containers/common/libimage"
 	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/pkg/compression"
 	"github.com/containers/image/v5/pkg/shortnames"
 	"github.com/containers/image/v5/transports"
 	"github.com/containers/image/v5/transports/alltransports"
@@ -318,6 +319,22 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin
 	pushOptions.SignBy = opts.SignBy
 	pushOptions.InsecureSkipTLSVerify = opts.SkipTLSVerify
 
+	compressionFormat := opts.CompressionFormat
+	if compressionFormat == "" {
+		config, err := ir.Libpod.GetConfigNoCopy()
+		if err != nil {
+			return "", err
+		}
+		compressionFormat = config.Engine.CompressionFormat
+	}
+	if compressionFormat != "" {
+		algo, err := compression.AlgorithmByName(compressionFormat)
+		if err != nil {
+			return "", err
+		}
+		pushOptions.CompressionFormat = &algo
+	}
+
 	if opts.All {
 		pushOptions.ImageListSelection = cp.CopyAllImages
 	}

pkg/domain/infra/tunnel/images.go

@@ -240,7 +240,7 @@ func (ir *ImageEngine) Import(ctx context.Context, opts entities.ImageImportOpti
 
 func (ir *ImageEngine) Push(ctx context.Context, source string, destination string, opts entities.ImagePushOptions) error {
 	options := new(images.PushOptions)
-	options.WithAll(opts.All).WithCompress(opts.Compress).WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithFormat(opts.Format).WithRemoveSignatures(opts.RemoveSignatures).WithQuiet(opts.Quiet)
+	options.WithAll(opts.All).WithCompress(opts.Compress).WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithFormat(opts.Format).WithRemoveSignatures(opts.RemoveSignatures).WithQuiet(opts.Quiet).WithCompressionFormat(opts.CompressionFormat)
 
 	if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined {
 		if s == types.OptionalBoolTrue {

pkg/domain/infra/tunnel/manifest.go

@@ -99,8 +99,7 @@ func (ir *ImageEngine) ManifestRm(ctx context.Context, names []string) (*entitie
 // ManifestPush pushes a manifest list or image index to the destination
 func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination string, opts entities.ImagePushOptions) (string, error) {
 	options := new(images.PushOptions)
-	options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures)
+	options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat)
-	options.WithAll(opts.All)
 
 	if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined {
 		if s == types.OptionalBoolTrue {

test/e2e/manifest_test.go

@@ -1,12 +1,14 @@
 package integration
 
 import (
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
 	podmanRegistry "github.com/containers/podman/v4/hack/podman-registry-go"
 	. "github.com/containers/podman/v4/test/utils"
+	"github.com/containers/storage/pkg/archive"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gexec"
@@ -292,6 +294,46 @@ var _ = Describe("Podman manifest", func() {
 			))
 	})
 
+	It("push with compression-format", func() {
+		SkipIfRemote("manifest push to dir not supported in remote mode")
+		session := podmanTest.Podman([]string{"manifest", "create", "foo"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		session = podmanTest.Podman([]string{"manifest", "add", "--all", "foo", imageList})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		dest := filepath.Join(podmanTest.TempDir, "pushed")
+		err := os.MkdirAll(dest, os.ModePerm)
+		Expect(err).To(BeNil())
+		defer func() {
+			os.RemoveAll(dest)
+		}()
+		session = podmanTest.Podman([]string{"push", "--compression-format=zstd", "foo", "oci:" + dest})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		foundZstdFile := false
+
+		blobsDir := filepath.Join(dest, "blobs", "sha256")
+
+		blobs, err := ioutil.ReadDir(blobsDir)
+		Expect(err).To(BeNil())
+
+		for _, f := range blobs {
+			blobPath := filepath.Join(blobsDir, f.Name())
+
+			sourceFile, err := ioutil.ReadFile(blobPath)
+			Expect(err).To(BeNil())
+
+			compressionType := archive.DetectCompression(sourceFile)
+			if compressionType == archive.Zstd {
+				foundZstdFile = true
+				break
+			}
+		}
+		Expect(foundZstdFile).To(BeTrue())
+	})
+
 	It("authenticated push", func() {
 		registryOptions := &podmanRegistry.Options{
 			Image: "docker-archive:" + imageTarPath(REGISTRY_IMAGE),