manifest/push: add support for --add-compression

Adds support for --add-compression which accepts multiple compression
formats and when used it will add all instances in a manifest list with
requested compression formats.

Signed-off-by: Aditya R <arajan@redhat.com>

cmd/podman/manifest/push.go

@@ -56,6 +56,10 @@ func init() {
 	flags.StringVar(&manifestPushOpts.Authfile, authfileFlagName, auth.GetDefaultAuthFile(), "path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override")
 	_ = pushCmd.RegisterFlagCompletionFunc(authfileFlagName, completion.AutocompleteDefault)
 
+	addCompressionFlagName := "add-compression"
+	flags.StringSliceVar(&manifestPushOpts.AddCompression, addCompressionFlagName, nil, "add instances with selected compression while pushing")
+	_ = pushCmd.RegisterFlagCompletionFunc(addCompressionFlagName, common.AutocompleteCompressionFormat)
+
 	certDirFlagName := "cert-dir"
 	flags.StringVar(&manifestPushOpts.CertDir, certDirFlagName, "", "use certificates at the specified path to access the registry")
 	_ = pushCmd.RegisterFlagCompletionFunc(certDirFlagName, completion.AutocompleteDefault)

docs/source/markdown/podman-manifest-push.1.md.in

@@ -14,6 +14,17 @@ The list image's ID and the digest of the image's manifest.
 
 ## OPTIONS
 
+#### **--add-compression**=*compression*
+
+Makes sure that requested compression variant for each platform is added to the manifest list keeping original instance
+intact in the same manifest list. Supported values are (`gzip`, `zstd` and `zstd:chunked`). Following flag can be used
+multiple times.
+
+Note that `--compression-format` controls the compression format of each instance in the manifest list. `--add-compression`
+will add another variant for each instance in the list with the specified compressions. `--compression-format` gzip `--add-compression`
+zstd will push a manifest list with each instance being compressed with gzip plus an additional variant of each instance
+being compressed with zstd.
+
 #### **--all**
 
 Push the images mentioned in the manifest list or image index, in addition to

pkg/api/handlers/libpod/manifests.go

@@ -340,6 +340,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 		RemoveSignatures  bool     `schema:"removeSignatures"`
 		TLSVerify         bool     `schema:"tlsVerify"`
 		Quiet             bool     `schema:"quiet"`
+		AddCompression    []string `schema:"addCompression"`
 	}{
 		// Add defaults here once needed.
 		TLSVerify: true,
@@ -373,6 +374,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 	options := entities.ImagePushOptions{
 		All:               query.All,
 		Authfile:          authfile,
+		AddCompression:    query.AddCompression,
 		CompressionFormat: query.CompressionFormat,
 		CompressionLevel:  query.CompressionLevel,
 		Format:            query.Format,

pkg/api/server/register_manifest.go

@@ -60,6 +60,13 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error {
 	//    type: string
 	//    required: true
 	//    description: the name or ID of the manifest list
+	//  - in: query
+	//    name: addCompression
+	//    required: false
+	//    description: add existing instances with requested compression algorithms to manifest list
+	//    type: array
+	//    items:
+	//      type: string
 	//  - in: path
 	//    name: destination
 	//    type: string

pkg/bindings/images/types.go

@@ -144,6 +144,8 @@ type PushOptions struct {
 	CompressionFormat *string
 	// CompressionLevel is the level to use for the compression of the blobs
 	CompressionLevel *int
+	// Add existing instances with requested compression algorithms to manifest list
+	AddCompression []string
 	// Manifest type of the pushed image
 	Format *string
 	// Password for authenticating against the registry.

pkg/bindings/images/types_push_options.go

@@ -93,6 +93,21 @@ func (o *PushOptions) GetCompressionLevel() int {
 	return *o.CompressionLevel
 }
 
+// WithAddCompression set field AddCompression to given value
+func (o *PushOptions) WithAddCompression(value []string) *PushOptions {
+	o.AddCompression = value
+	return o
+}
+
+// GetAddCompression returns value of field AddCompression
+func (o *PushOptions) GetAddCompression() []string {
+	if o.AddCompression == nil {
+		var z []string
+		return z
+	}
+	return o.AddCompression
+}
+
 // WithFormat set field Format to given value
 func (o *PushOptions) WithFormat(value string) *PushOptions {
 	o.Format = &value

pkg/domain/entities/images.go

@@ -244,6 +244,9 @@ type ImagePushOptions struct {
 	// integers in the slice represent 0-indexed layer indices, with support for negative
 	// indexing. i.e. 0 is the first layer, -1 is the last (top-most) layer.
 	OciEncryptLayers *[]int
+	//  If necessary, add clones of existing instances with requested compression algorithms to manifest list
+	// Note: Following option is only valid for `manifest push`
+	AddCompression []string
 }
 
 // ImagePushReport is the response from pushing an image.

pkg/domain/infra/abi/manifest.go

@@ -345,6 +345,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin
 	pushOptions.InsecureSkipTLSVerify = opts.SkipTLSVerify
 	pushOptions.Writer = opts.Writer
 	pushOptions.CompressionLevel = opts.CompressionLevel
+	pushOptions.AddCompression = opts.AddCompression
 
 	compressionFormat := opts.CompressionFormat
 	if compressionFormat == "" {

pkg/domain/infra/tunnel/manifest.go

@@ -135,7 +135,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin
 	}
 
 	options := new(images.PushOptions)
-	options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer)
+	options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer).WithAddCompression(opts.AddCompression)
 
 	if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined {
 		if s == types.OptionalBoolTrue {

test/e2e/manifest_test.go

@@ -13,8 +13,28 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gexec"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+// Internal function to verify instance compression
+func verifyInstanceCompression(descriptor []imgspecv1.Descriptor, compression string, arch string) bool {
+	for _, instance := range descriptor {
+		if instance.Platform.Architecture != arch {
+			continue
+		}
+		if compression == "zstd" {
+			// if compression is zstd annotations must contain
+			val, ok := instance.Annotations["io.github.containers.compression.zstd"]
+			if ok && val == "true" {
+				return true
+			}
+		} else if len(instance.Annotations) == 0 {
+			return true
+		}
+	}
+	return false
+}
+
 var _ = Describe("Podman manifest", func() {
 
 	const (
@@ -135,6 +155,63 @@ var _ = Describe("Podman manifest", func() {
 		Expect(session2.OutputToString()).To(Equal(session.OutputToString()))
 	})
 
+	It("push with --add-compression", func() {
+		if podmanTest.Host.Arch == "ppc64le" {
+			Skip("No registry image for ppc64le")
+		}
+		if isRootless() {
+			err := podmanTest.RestoreArtifact(REGISTRY_IMAGE)
+			Expect(err).ToNot(HaveOccurred())
+		}
+		lock := GetPortLock("5000")
+		defer lock.Unlock()
+		session := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", REGISTRY_IMAGE, "/entrypoint.sh", "/etc/docker/registry/config.yml"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		if !WaitContainerReady(podmanTest, "registry", "listening on", 20, 1) {
+			Skip("Cannot start docker registry.")
+		}
+
+		session = podmanTest.Podman([]string{"build", "--platform", "linux/amd64", "-t", "imageone", "build/basicalpine"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"build", "--platform", "linux/arm64", "-t", "imagetwo", "build/basicalpine"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"manifest", "create", "foobar"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		session = podmanTest.Podman([]string{"manifest", "add", "foobar", "containers-storage:localhost/imageone:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		session = podmanTest.Podman([]string{"manifest", "add", "foobar", "containers-storage:localhost/imagetwo:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		push := podmanTest.Podman([]string{"manifest", "push", "--all", "--add-compression", "zstd", "--tls-verify=false", "--remove-signatures", "foobar", "localhost:5000/list"})
+		push.WaitWithDefaultTimeout()
+		Expect(push).Should(Exit(0))
+		output := push.ErrorToString()
+		// 4 images must be pushed two for gzip and two for zstd
+		Expect(output).To(ContainSubstring("Copying 4 images generated from 2 images in list"))
+
+		session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/list:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		var index imgspecv1.Index
+		inspectData := []byte(session.OutputToString())
+		err := json.Unmarshal(inspectData, &index)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(verifyInstanceCompression(index.Manifests, "zstd", "amd64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeTrue())
+	})
+
 	It("add --all", func() {
 		session := podmanTest.Podman([]string{"manifest", "create", "foo"})
 		session.WaitWithDefaultTimeout()

test/system/012-manifest.bats

@@ -4,6 +4,39 @@ load helpers
 load helpers.network
 load helpers.registry
 
+# Helper function for several of the tests which verifies compression.
+#
+#  Usage:  validate_instance_compression INDEX MANIFEST ARCH COMPRESSION
+#
+#     INDEX             instance which needs to be verified in
+#                       provided manifest list.
+#
+#     MANIFEST          OCI manifest specification in json format
+#
+#     ARCH              instance architecture
+#
+#     COMPRESSION       compression algorithm name; e.g "zstd".
+#
+function validate_instance_compression {
+  case $4 in
+
+   gzip)
+    run jq -r '.manifests['$1'].annotations' <<< $2
+    # annotation is `null` for gzip compression
+    assert "$output" = "null" ".manifests[$1].annotations (null means gzip)"
+    ;;
+
+  zstd)
+    # annotation `'"io.github.containers.compression.zstd": "true"'` must be there for zstd compression
+    run jq -r '.manifests['$1'].annotations."io.github.containers.compression.zstd"' <<< $2
+    assert "$output" = "true" ".manifests[$1].annotations.'io.github.containers.compression.zstd' (io.github.containers.compression.zstd must be set)"
+    ;;
+  esac
+
+  run jq -r '.manifests['$1'].platform.architecture' <<< $2
+  assert "$output" = $3 ".manifests[$1].platform.architecture"
+}
+
 # Regression test for #8931
 @test "podman images - bare manifest list" {
     # Create an empty manifest list and list images.
@@ -56,4 +89,46 @@ load helpers.registry
     is "$output" ".*\"mediaType\": \"application/vnd.docker.distribution.manifest.list.v2+json\"" "Verify --tls-verify=false with REGISTRY_AUTH_FILE works against an insecure registry"
 }
 
+@test "manifest list --add-compression with zstd" {
+    if ! type -p skopeo; then
+        skip "skopeo not available"
+    fi
+    skip_if_remote "running a local registry doesn't work with podman-remote"
+    start_registry
+
+    tmpdir=$PODMAN_TMPDIR/build-test
+    mkdir -p $tmpdir
+    dockerfile=$tmpdir/Dockerfile
+    cat >$dockerfile <<EOF
+FROM alpine
+EOF
+    authfile=${PODMAN_LOGIN_WORKDIR}/auth-$(random_string 10).json
+    run_podman login --tls-verify=false \
+               --username ${PODMAN_LOGIN_USER} \
+               --password-stdin \
+               --authfile=$authfile \
+               localhost:${PODMAN_LOGIN_REGISTRY_PORT} <<<"${PODMAN_LOGIN_PASS}"
+    is "$output" "Login Succeeded!" "output from podman login"
+
+  manifest1="localhost:${PODMAN_LOGIN_REGISTRY_PORT}/test:1.0"
+  run_podman build -t image1 --platform linux/amd64 -f $dockerfile
+  run_podman build -t image2 --platform linux/arm64 -f $dockerfile
+
+  run_podman manifest create foo
+  run_podman images -a
+  run_podman manifest add foo containers-storage:localhost/image1:latest
+  run_podman manifest add foo containers-storage:localhost/image2:latest
+
+  run_podman manifest push --authfile=$authfile --all --add-compression zstd --tls-verify=false foo $manifest1
+
+  run skopeo inspect --authfile=$authfile --tls-verify=false --raw docker://$manifest1
+  echo $output
+  list="$output"
+
+  validate_instance_compression "0" "$list" "amd64" "gzip"
+  validate_instance_compression "1" "$list" "arm64" "gzip"
+  validate_instance_compression "2" "$list" "amd64" "zstd"
+  validate_instance_compression "3" "$list" "arm64" "zstd"
+}
+
 # vim: filetype=sh