Merge pull request #18940 from giuseppe/add-compression-level

cmd, push: expose --compression-level

cmd/podman/images/push.go

@@ -130,6 +130,10 @@ func pushFlags(cmd *cobra.Command) {
 	flags.StringVar(&pushOptions.CompressionFormat, compressionFormat, "", "compression format to use")
 	_ = cmd.RegisterFlagCompletionFunc(compressionFormat, common.AutocompleteCompressionFormat)
 
+	compressionLevel := "compression-level"
+	flags.Int(compressionLevel, 0, "compression level to use")
+	_ = cmd.RegisterFlagCompletionFunc(compressionLevel, completion.AutocompleteNone)
+
 	encryptionKeysFlagName := "encryption-key"
 	flags.StringSliceVar(&pushOptions.EncryptionKeys, encryptionKeysFlagName, nil, "Key with the encryption protocol to use to encrypt the image (e.g. jwe:/path/to/key.pem)")
 	_ = cmd.RegisterFlagCompletionFunc(encryptionKeysFlagName, completion.AutocompleteDefault)
@@ -201,6 +205,14 @@ func imagePush(cmd *cobra.Command, args []string) error {
 	pushOptions.OciEncryptConfig = encConfig
 	pushOptions.OciEncryptLayers = encLayers
 
+	if cmd.Flags().Changed("compression-level") {
+		val, err := cmd.Flags().GetInt("compression-level")
+		if err != nil {
+			return err
+		}
+		pushOptions.CompressionLevel = &val
+	}
+
 	// Let's do all the remaining Yoga in the API to prevent us from scattering
 	// logic across (too) many parts of the code.
 	report, err := registry.ImageEngine().Push(registry.GetContext(), source, destination, pushOptions.ImagePushOptions)

cmd/podman/manifest/push.go

@@ -100,6 +100,10 @@ func init() {
 	flags.StringVar(&manifestPushOpts.CompressionFormat, compressionFormat, "", "compression format to use")
 	_ = pushCmd.RegisterFlagCompletionFunc(compressionFormat, common.AutocompleteCompressionFormat)
 
+	compressionLevel := "compression-level"
+	flags.Int(compressionLevel, 0, "compression level to use")
+	_ = pushCmd.RegisterFlagCompletionFunc(compressionLevel, completion.AutocompleteNone)
+
 	if registry.IsRemote() {
 		_ = flags.MarkHidden("cert-dir")
 		_ = flags.MarkHidden(signByFlagName)
@@ -155,6 +159,15 @@ func push(cmd *cobra.Command, args []string) error {
 		}
 		manifestPushOpts.SkipTLSVerify = types.NewOptionalBool(manifestPushOpts.Insecure)
 	}
+
+	if cmd.Flags().Changed("compression-level") {
+		val, err := cmd.Flags().GetInt("compression-level")
+		if err != nil {
+			return err
+		}
+		manifestPushOpts.CompressionLevel = &val
+	}
+
 	digest, err := registry.ImageEngine().ManifestPush(registry.Context(), listImageSpec, destSpec, manifestPushOpts.ImagePushOptions)
 	if err != nil {
 		return err

docs/source/markdown/options/compression-level.md

@@ -0,0 +1,7 @@
+####> This option file is used in:
+####>   podman manifest push, push
+####> If file is edited, make sure the changes
+####> are applicable to all of those.
+#### **--compression-level**=*level*
+
+Specifies the compression level to use.  The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive) with a default of 3, while for gzip it is 1-9 (inclusive) and has a default of 5.

docs/source/markdown/podman-manifest-push.1.md.in

@@ -25,6 +25,8 @@ the list or index itself. (Default true)
 
 @@option compression-format
 
+@@option compression-level
+
 @@option creds
 
 @@option digestfile

docs/source/markdown/podman-push.1.md.in

@@ -54,6 +54,8 @@ Note: This flag can only be set when using the **dir** transport
 
 @@option compression-format
 
+@@option compression-level
+
 @@option creds
 
 @@option digestfile

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/containernetworking/cni v1.1.2
 	github.com/containernetworking/plugins v1.3.0
 	github.com/containers/buildah v1.30.1-0.20230504052500-e925b5852e07
-	github.com/containers/common v0.53.1-0.20230620132900-ac2475afa81d
+	github.com/containers/common v0.53.1-0.20230621115248-a2cd3ea30337
 	github.com/containers/conmon v2.0.20+incompatible
 	github.com/containers/image/v5 v5.25.1-0.20230613183705-07ced6137083
 	github.com/containers/libhvee v0.0.5

go.sum

@@ -239,8 +239,8 @@ github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q
 github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0=
 github.com/containers/buildah v1.30.1-0.20230504052500-e925b5852e07 h1:Bs2sNFh/fSYr4J6JJLFqzyn3dp6HhlA6ewFwRYUpeIE=
 github.com/containers/buildah v1.30.1-0.20230504052500-e925b5852e07/go.mod h1:6A/BK0YJLXL8+AqlbceKJrhUT+NtEgsvAc51F7TAllc=
-github.com/containers/common v0.53.1-0.20230620132900-ac2475afa81d h1:Z+xdHWSwjW/VGdYGufKfqji+G7FQ1IdkFai0MOpqzd4=
+github.com/containers/common v0.53.1-0.20230621115248-a2cd3ea30337 h1:Z9wxp08tzCKgI3ziVwpoMyQcDKH8z9VmgyeHJcnunj4=
-github.com/containers/common v0.53.1-0.20230620132900-ac2475afa81d/go.mod h1:qE1MzGl69IoK7ZNCCH51+aLVjyQtnH0LiZe0wG32Jy0=
+github.com/containers/common v0.53.1-0.20230621115248-a2cd3ea30337/go.mod h1:qE1MzGl69IoK7ZNCCH51+aLVjyQtnH0LiZe0wG32Jy0=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
 github.com/containers/image/v5 v5.25.1-0.20230613183705-07ced6137083 h1:6Pbnll97ls6G0U3DSxaTqp7Sd8Fykc4gd7BUJm7Bpn8=

pkg/api/handlers/libpod/images_push.go

@@ -27,6 +27,7 @@ func PushImage(w http.ResponseWriter, r *http.Request) {
 	query := struct {
 		All               bool   `schema:"all"`
 		CompressionFormat string `schema:"compressionFormat"`
+		CompressionLevel  *int   `schema:"compressionLevel"`
 		Destination       string `schema:"destination"`
 		Format            string `schema:"format"`
 		RemoveSignatures  bool   `schema:"removeSignatures"`
@@ -75,6 +76,7 @@ func PushImage(w http.ResponseWriter, r *http.Request) {
 		All:               query.All,
 		Authfile:          authfile,
 		CompressionFormat: query.CompressionFormat,
+		CompressionLevel:  query.CompressionLevel,
 		Format:            query.Format,
 		Password:          password,
 		Quiet:             true,

pkg/api/handlers/libpod/manifests.go

@@ -328,6 +328,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 	query := struct {
 		All               bool   `schema:"all"`
 		CompressionFormat string `schema:"compressionFormat"`
+		CompressionLevel  *int   `schema:"compressionLevel"`
 		Format            string `schema:"format"`
 		RemoveSignatures  bool   `schema:"removeSignatures"`
 		TLSVerify         bool   `schema:"tlsVerify"`
@@ -366,6 +367,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 		All:               query.All,
 		Authfile:          authfile,
 		CompressionFormat: query.CompressionFormat,
+		CompressionLevel:  query.CompressionLevel,
 		Format:            query.Format,
 		Password:          password,
 		Quiet:             true,

pkg/bindings/images/types.go

@@ -142,6 +142,8 @@ type PushOptions struct {
 	Compress *bool
 	// CompressionFormat is the format to use for the compression of the blobs
 	CompressionFormat *string
+	// CompressionLevel is the level to use for the compression of the blobs
+	CompressionLevel *int
 	// Manifest type of the pushed image
 	Format *string
 	// Password for authenticating against the registry.

pkg/bindings/images/types_push_options.go

@@ -78,6 +78,21 @@ func (o *PushOptions) GetCompressionFormat() string {
 	return *o.CompressionFormat
 }
 
+// WithCompressionLevel set field CompressionLevel to given value
+func (o *PushOptions) WithCompressionLevel(value int) *PushOptions {
+	o.CompressionLevel = &value
+	return o
+}
+
+// GetCompressionLevel returns value of field CompressionLevel
+func (o *PushOptions) GetCompressionLevel() int {
+	if o.CompressionLevel == nil {
+		var z int
+		return z
+	}
+	return *o.CompressionLevel
+}
+
 // WithFormat set field Format to given value
 func (o *PushOptions) WithFormat(value string) *PushOptions {
 	o.Format = &value

pkg/domain/entities/images.go

@@ -231,6 +231,8 @@ type ImagePushOptions struct {
 	Progress chan types.ProgressProperties
 	// CompressionFormat is the format to use for the compression of the blobs
 	CompressionFormat string
+	// CompressionLevel is the level to use for the compression of the blobs
+	CompressionLevel *int
 	// Writer is used to display copy information including progress bars.
 	Writer io.Writer
 	// OciEncryptConfig when non-nil indicates that an image should be encrypted.

pkg/domain/infra/abi/images.go

@@ -316,6 +316,7 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri
 	pushOptions.Writer = options.Writer
 	pushOptions.OciEncryptConfig = options.OciEncryptConfig
 	pushOptions.OciEncryptLayers = options.OciEncryptLayers
+	pushOptions.CompressionLevel = options.CompressionLevel
 
 	compressionFormat := options.CompressionFormat
 	if compressionFormat == "" {
@@ -333,6 +334,14 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri
 		pushOptions.CompressionFormat = &algo
 	}
 
+	if pushOptions.CompressionLevel == nil {
+		config, err := ir.Libpod.GetConfigNoCopy()
+		if err != nil {
+			return nil, err
+		}
+		pushOptions.CompressionLevel = config.Engine.CompressionLevel
+	}
+
 	if !options.Quiet && pushOptions.Writer == nil {
 		pushOptions.Writer = os.Stderr
 	}

pkg/domain/infra/abi/manifest.go

@@ -340,6 +340,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin
 	pushOptions.SignSigstorePrivateKeyPassphrase = opts.SignSigstorePrivateKeyPassphrase
 	pushOptions.InsecureSkipTLSVerify = opts.SkipTLSVerify
 	pushOptions.Writer = opts.Writer
+	pushOptions.CompressionLevel = opts.CompressionLevel
 
 	compressionFormat := opts.CompressionFormat
 	if compressionFormat == "" {
@@ -356,6 +357,13 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin
 		}
 		pushOptions.CompressionFormat = &algo
 	}
+	if pushOptions.CompressionLevel == nil {
+		config, err := ir.Libpod.GetConfigNoCopy()
+		if err != nil {
+			return "", err
+		}
+		pushOptions.CompressionLevel = config.Engine.CompressionLevel
+	}
 
 	if opts.All {
 		pushOptions.ImageListSelection = cp.CopyAllImages

pkg/domain/infra/tunnel/images.go

@@ -254,6 +254,10 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri
 	options := new(images.PushOptions)
 	options.WithAll(opts.All).WithCompress(opts.Compress).WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithFormat(opts.Format).WithRemoveSignatures(opts.RemoveSignatures).WithQuiet(opts.Quiet).WithCompressionFormat(opts.CompressionFormat).WithProgressWriter(opts.Writer)
 
+	if opts.CompressionLevel != nil {
+		options.WithCompressionLevel(*opts.CompressionLevel)
+	}
+
 	if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined {
 		if s == types.OptionalBoolTrue {
 			options.WithSkipTLSVerify(true)

test/e2e/manifest_test.go

@@ -319,14 +319,33 @@ var _ = Describe("Podman manifest", func() {
 			))
 	})
 
-	It("push with compression-format", func() {
+	It("push with compression-format and compression-level", func() {
 		SkipIfRemote("manifest push to dir not supported in remote mode")
-		session := podmanTest.Podman([]string{"manifest", "create", "foo"})
+		session := podmanTest.Podman([]string{"pull", ALPINE})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(Exit(0))
-		session = podmanTest.Podman([]string{"manifest", "add", "--all", "foo", imageList})
+
+		dockerfile := `FROM quay.io/libpod/alpine:latest
+RUN touch /file
+`
+		podmanTest.BuildImage(dockerfile, "localhost/test", "false")
+
+		session = podmanTest.Podman([]string{"manifest", "create", "foo"})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"manifest", "add", "foo", "containers-storage:localhost/test"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		// Invalid compression format specified, it must fail
+		tmpDir := filepath.Join(podmanTest.TempDir, "wrong-compression")
+		session = podmanTest.Podman([]string{"manifest", "push", "--compression-format", "gzip", "--compression-level", "50", "foo", "oci:" + tmpDir})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(125))
+		output := session.ErrorToString()
+		Expect(output).To(ContainSubstring("invalid compression level"))
+
 		dest := filepath.Join(podmanTest.TempDir, "pushed")
 		err := os.MkdirAll(dest, os.ModePerm)
 		Expect(err).ToNot(HaveOccurred())
@@ -434,7 +453,7 @@ var _ = Describe("Podman manifest", func() {
 		Expect(output).To(ContainSubstring("Writing manifest to image destination"))
 		Expect(output).To(ContainSubstring("Storing signatures"))
 
-		push = podmanTest.Podman([]string{"manifest", "push", "--tls-verify=false", "--creds=podmantest:wrongpasswd", "foo", "localhost:" + registry.Port + "/credstest"})
+		push = podmanTest.Podman([]string{"manifest", "push", "--compression-format=gzip", "--compression-level=2", "--tls-verify=false", "--creds=podmantest:wrongpasswd", "foo", "localhost:" + registry.Port + "/credstest"})
 		push.WaitWithDefaultTimeout()
 		Expect(push).To(ExitWithError())
 		Expect(push.ErrorToString()).To(ContainSubstring(": authentication required"))

test/e2e/push_test.go

@@ -46,10 +46,18 @@ var _ = Describe("Podman push", func() {
 		Expect(session).Should(Exit(0))
 	})
 
-	It("podman push to oci with compression-format", func() {
+	It("podman push to oci with compression-format and compression-level", func() {
 		SkipIfRemote("Remote push does not support dir transport")
 		bbdir := filepath.Join(podmanTest.TempDir, "busybox-oci")
-		session := podmanTest.Podman([]string{"push", "--compression-format=zstd", "--remove-signatures", ALPINE,
+
+		// Invalid compression format specified, it must fail
+		session := podmanTest.Podman([]string{"push", "--compression-format=gzip", "--compression-level=40", ALPINE, fmt.Sprintf("oci:%s", bbdir)})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(125))
+		output := session.ErrorToString()
+		Expect(output).To(ContainSubstring("invalid compression level"))
+
+		session = podmanTest.Podman([]string{"push", "--compression-format=zstd", "--remove-signatures", ALPINE,
 			fmt.Sprintf("oci:%s", bbdir)})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(Exit(0))
@@ -99,7 +107,7 @@ var _ = Describe("Podman push", func() {
 		Expect(push).Should(Exit(0))
 		Expect(push.ErrorToString()).To(BeEmpty())
 
-		push = podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"})
+		push = podmanTest.Podman([]string{"push", "--compression-format=gzip", "--compression-level=1", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"})
 		push.WaitWithDefaultTimeout()
 		Expect(push).Should(Exit(0))
 		output := push.ErrorToString()

vendor/github.com/containers/common/pkg/config/config.go

@@ -510,6 +510,9 @@ type EngineConfig struct {
 
 	// CompressionFormat is the compression format used to compress image layers.
 	CompressionFormat string `toml:"compression_format,omitempty"`
+
+	// CompressionLevel is the compression level used to compress image layers.
+	CompressionLevel *int `toml:"compression_level,omitempty"`
 }
 
 // SetOptions contains a subset of options in a Config. It's used to indicate if

vendor/github.com/containers/common/pkg/config/containers.conf

@@ -381,6 +381,12 @@ default_sysctls = [
 #
 #compression_format = "gzip"
 
+# The compression level to use when pushing an image.
+# Valid options depend on the compression format used.
+# For gzip, valid options are 1-9, with a default of 5.
+# For zstd, valid options are 1-20, with a default of 3.
+#
+#compression_level = 5
 
 # Cgroup management implementation used for the runtime.
 # Valid options "systemd" or "cgroupfs"

vendor/github.com/containers/common/pkg/config/containers.conf-freebsd

@@ -311,6 +311,13 @@ default_sysctls = [
 #
 #compression_format = "gzip"
 
+# The compression level to use when pushing an image.
+# Valid options depend on the compression format used.
+# For gzip, valid options are 1-9, with a default of 5.
+# For zstd, valid options are 1-20, with a default of 3.
+#
+#compression_level = 5
+
 # Environment variables to pass into conmon
 #
 #conmon_env_vars = [

vendor/modules.txt

@@ -125,7 +125,7 @@ github.com/containers/buildah/pkg/rusage
 github.com/containers/buildah/pkg/sshagent
 github.com/containers/buildah/pkg/util
 github.com/containers/buildah/util
-# github.com/containers/common v0.53.1-0.20230620132900-ac2475afa81d
+# github.com/containers/common v0.53.1-0.20230621115248-a2cd3ea30337
 ## explicit; go 1.18
 github.com/containers/common/libimage
 github.com/containers/common/libimage/define