Merge pull request #19468 from flouthoc/manifest-add-compression

manifest, push: implement `--add-compression` to push with compressed variants.

cmd/podman/manifest/push.go

@@ -56,6 +56,10 @@ func init() {
 	flags.StringVar(&manifestPushOpts.Authfile, authfileFlagName, auth.GetDefaultAuthFile(), "path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override")
 	_ = pushCmd.RegisterFlagCompletionFunc(authfileFlagName, completion.AutocompleteDefault)
 
+	addCompressionFlagName := "add-compression"
+	flags.StringSliceVar(&manifestPushOpts.AddCompression, addCompressionFlagName, nil, "add instances with selected compression while pushing")
+	_ = pushCmd.RegisterFlagCompletionFunc(addCompressionFlagName, common.AutocompleteCompressionFormat)
+
 	certDirFlagName := "cert-dir"
 	flags.StringVar(&manifestPushOpts.CertDir, certDirFlagName, "", "use certificates at the specified path to access the registry")
 	_ = pushCmd.RegisterFlagCompletionFunc(certDirFlagName, completion.AutocompleteDefault)

docs/source/markdown/podman-manifest-push.1.md.in

@@ -14,6 +14,17 @@ The list image's ID and the digest of the image's manifest.
 
 ## OPTIONS
 
+#### **--add-compression**=*compression*
+
+Makes sure that requested compression variant for each platform is added to the manifest list keeping original instance
+intact in the same manifest list. Supported values are (`gzip`, `zstd` and `zstd:chunked`). Following flag can be used
+multiple times.
+
+Note that `--compression-format` controls the compression format of each instance in the manifest list. `--add-compression`
+will add another variant for each instance in the list with the specified compressions. `--compression-format` gzip `--add-compression`
+zstd will push a manifest list with each instance being compressed with gzip plus an additional variant of each instance
+being compressed with zstd.
+
 #### **--all**
 
 Push the images mentioned in the manifest list or image index, in addition to

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/containers/buildah v1.31.1-0.20230722114901-5ece066f82c6
 	github.com/containers/common v0.55.1-0.20230801150045-44bfd82e3ed2
 	github.com/containers/conmon v2.0.20+incompatible
-	github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac
+	github.com/containers/image/v5 v5.26.1-0.20230801083106-fcf7f0e1712a
 	github.com/containers/libhvee v0.4.0
 	github.com/containers/ocicrypt v1.1.7
 	github.com/containers/psgo v1.8.0
@@ -90,7 +90,7 @@ require (
 	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
 	github.com/coreos/go-oidc/v3 v3.6.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
-	github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744 // indirect
+	github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect
 	github.com/disiqueira/gotree/v3 v3.0.2 // indirect

go.sum

@@ -250,8 +250,8 @@ github.com/containers/common v0.55.1-0.20230801150045-44bfd82e3ed2 h1:E6OuqpGlvU
 github.com/containers/common v0.55.1-0.20230801150045-44bfd82e3ed2/go.mod h1:VlEW0hd11FqVMbWYYjuDCU1+IEqElPZO1RznrRHkxyQ=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
-github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac h1:EQQX+EO+F30H9vJS6vfDXx83Z6OL1YNkO5LN36BtPnM=
-github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac/go.mod h1:Zg7m6YHPZRl/wbUDZ6vt+yAyXAjAvALVUelmsIPpMcE=
+github.com/containers/image/v5 v5.26.1-0.20230801083106-fcf7f0e1712a h1:ZK6GNc7wWP9/CTQySx0TM9VN9p+og4Pfd3Y5aAHrwLk=
+github.com/containers/image/v5 v5.26.1-0.20230801083106-fcf7f0e1712a/go.mod h1:vsetwKSm1kQayKIWlN7SdGNu/KwcVCgnrhh4Z6Yb75s=
 github.com/containers/libhvee v0.4.0 h1:HGHIIExgP2PjwjHKKoQM3B+3qakNIZcmmkiAO4luAZE=
 github.com/containers/libhvee v0.4.0/go.mod h1:fyWDxNQccveTdE3Oe+QRuLbwF/iyV0hDxXqRX5Svlic=
 github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
@@ -297,8 +297,8 @@ github.com/crc-org/vfkit v0.1.1/go.mod h1:vjZiHDacUi0iLosvwyLvqJvJXQhByzlLQbMkdI
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
-github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744 h1:MqMnhqqfDsYF2bjxndKIqvISTIRBb1KCzrIwVzKJHe0=
-github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
+github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd h1:0av0vtcjA8Hqv5gyWj79CLCFVwOOyBNWPjrfUWceMNg=
+github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
 github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=

pkg/api/handlers/libpod/manifests.go

@@ -333,13 +333,14 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 	decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)
 
 	query := struct {
-		All               bool   `schema:"all"`
-		CompressionFormat string `schema:"compressionFormat"`
-		CompressionLevel  *int   `schema:"compressionLevel"`
-		Format            string `schema:"format"`
-		RemoveSignatures  bool   `schema:"removeSignatures"`
-		TLSVerify         bool   `schema:"tlsVerify"`
-		Quiet             bool   `schema:"quiet"`
+		All               bool     `schema:"all"`
+		CompressionFormat string   `schema:"compressionFormat"`
+		CompressionLevel  *int     `schema:"compressionLevel"`
+		Format            string   `schema:"format"`
+		RemoveSignatures  bool     `schema:"removeSignatures"`
+		TLSVerify         bool     `schema:"tlsVerify"`
+		Quiet             bool     `schema:"quiet"`
+		AddCompression    []string `schema:"addCompression"`
 	}{
 		// Add defaults here once needed.
 		TLSVerify: true,
@@ -373,6 +374,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) {
 	options := entities.ImagePushOptions{
 		All:               query.All,
 		Authfile:          authfile,
+		AddCompression:    query.AddCompression,
 		CompressionFormat: query.CompressionFormat,
 		CompressionLevel:  query.CompressionLevel,
 		Format:            query.Format,

pkg/api/server/register_manifest.go

@@ -60,6 +60,13 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error {
 	//    type: string
 	//    required: true
 	//    description: the name or ID of the manifest list
+	//  - in: query
+	//    name: addCompression
+	//    required: false
+	//    description: add existing instances with requested compression algorithms to manifest list
+	//    type: array
+	//    items:
+	//      type: string
 	//  - in: path
 	//    name: destination
 	//    type: string

pkg/bindings/images/types.go

@@ -144,6 +144,8 @@ type PushOptions struct {
 	CompressionFormat *string
 	// CompressionLevel is the level to use for the compression of the blobs
 	CompressionLevel *int
+	// Add existing instances with requested compression algorithms to manifest list
+	AddCompression []string
 	// Manifest type of the pushed image
 	Format *string
 	// Password for authenticating against the registry.

pkg/bindings/images/types_push_options.go

@@ -93,6 +93,21 @@ func (o *PushOptions) GetCompressionLevel() int {
 	return *o.CompressionLevel
 }
 
+// WithAddCompression set field AddCompression to given value
+func (o *PushOptions) WithAddCompression(value []string) *PushOptions {
+	o.AddCompression = value
+	return o
+}
+
+// GetAddCompression returns value of field AddCompression
+func (o *PushOptions) GetAddCompression() []string {
+	if o.AddCompression == nil {
+		var z []string
+		return z
+	}
+	return o.AddCompression
+}
+
 // WithFormat set field Format to given value
 func (o *PushOptions) WithFormat(value string) *PushOptions {
 	o.Format = &value

pkg/domain/entities/images.go

@@ -244,6 +244,9 @@ type ImagePushOptions struct {
 	// integers in the slice represent 0-indexed layer indices, with support for negative
 	// indexing. i.e. 0 is the first layer, -1 is the last (top-most) layer.
 	OciEncryptLayers *[]int
+	//  If necessary, add clones of existing instances with requested compression algorithms to manifest list
+	// Note: Following option is only valid for `manifest push`
+	AddCompression []string
 }
 
 // ImagePushReport is the response from pushing an image.

pkg/domain/infra/abi/manifest.go

@@ -345,6 +345,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin
 	pushOptions.InsecureSkipTLSVerify = opts.SkipTLSVerify
 	pushOptions.Writer = opts.Writer
 	pushOptions.CompressionLevel = opts.CompressionLevel
+	pushOptions.AddCompression = opts.AddCompression
 
 	compressionFormat := opts.CompressionFormat
 	if compressionFormat == "" {

pkg/domain/infra/tunnel/manifest.go

@@ -135,7 +135,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin
 	}
 
 	options := new(images.PushOptions)
-	options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer)
+	options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer).WithAddCompression(opts.AddCompression)
 
 	if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined {
 		if s == types.OptionalBoolTrue {

test/e2e/manifest_test.go

@@ -13,8 +13,28 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gexec"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+// Internal function to verify instance compression
+func verifyInstanceCompression(descriptor []imgspecv1.Descriptor, compression string, arch string) bool {
+	for _, instance := range descriptor {
+		if instance.Platform.Architecture != arch {
+			continue
+		}
+		if compression == "zstd" {
+			// if compression is zstd annotations must contain
+			val, ok := instance.Annotations["io.github.containers.compression.zstd"]
+			if ok && val == "true" {
+				return true
+			}
+		} else if len(instance.Annotations) == 0 {
+			return true
+		}
+	}
+	return false
+}
+
 var _ = Describe("Podman manifest", func() {
 
 	const (
@@ -135,6 +155,63 @@ var _ = Describe("Podman manifest", func() {
 		Expect(session2.OutputToString()).To(Equal(session.OutputToString()))
 	})
 
+	It("push with --add-compression", func() {
+		if podmanTest.Host.Arch == "ppc64le" {
+			Skip("No registry image for ppc64le")
+		}
+		if isRootless() {
+			err := podmanTest.RestoreArtifact(REGISTRY_IMAGE)
+			Expect(err).ToNot(HaveOccurred())
+		}
+		lock := GetPortLock("5000")
+		defer lock.Unlock()
+		session := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", REGISTRY_IMAGE, "/entrypoint.sh", "/etc/docker/registry/config.yml"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		if !WaitContainerReady(podmanTest, "registry", "listening on", 20, 1) {
+			Skip("Cannot start docker registry.")
+		}
+
+		session = podmanTest.Podman([]string{"build", "--platform", "linux/amd64", "-t", "imageone", "build/basicalpine"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"build", "--platform", "linux/arm64", "-t", "imagetwo", "build/basicalpine"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"manifest", "create", "foobar"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		session = podmanTest.Podman([]string{"manifest", "add", "foobar", "containers-storage:localhost/imageone:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		session = podmanTest.Podman([]string{"manifest", "add", "foobar", "containers-storage:localhost/imagetwo:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		push := podmanTest.Podman([]string{"manifest", "push", "--all", "--add-compression", "zstd", "--tls-verify=false", "--remove-signatures", "foobar", "localhost:5000/list"})
+		push.WaitWithDefaultTimeout()
+		Expect(push).Should(Exit(0))
+		output := push.ErrorToString()
+		// 4 images must be pushed two for gzip and two for zstd
+		Expect(output).To(ContainSubstring("Copying 4 images generated from 2 images in list"))
+
+		session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/list:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		var index imgspecv1.Index
+		inspectData := []byte(session.OutputToString())
+		err := json.Unmarshal(inspectData, &index)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(verifyInstanceCompression(index.Manifests, "zstd", "amd64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeTrue())
+		Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeTrue())
+	})
+
 	It("add --all", func() {
 		session := podmanTest.Podman([]string{"manifest", "create", "foo"})
 		session.WaitWithDefaultTimeout()

test/system/012-manifest.bats

@@ -4,6 +4,39 @@ load helpers
 load helpers.network
 load helpers.registry
 
+# Helper function for several of the tests which verifies compression.
+#
+#  Usage:  validate_instance_compression INDEX MANIFEST ARCH COMPRESSION
+#
+#     INDEX             instance which needs to be verified in
+#                       provided manifest list.
+#
+#     MANIFEST          OCI manifest specification in json format
+#
+#     ARCH              instance architecture
+#
+#     COMPRESSION       compression algorithm name; e.g "zstd".
+#
+function validate_instance_compression {
+  case $4 in
+
+   gzip)
+    run jq -r '.manifests['$1'].annotations' <<< $2
+    # annotation is `null` for gzip compression
+    assert "$output" = "null" ".manifests[$1].annotations (null means gzip)"
+    ;;
+
+  zstd)
+    # annotation `'"io.github.containers.compression.zstd": "true"'` must be there for zstd compression
+    run jq -r '.manifests['$1'].annotations."io.github.containers.compression.zstd"' <<< $2
+    assert "$output" = "true" ".manifests[$1].annotations.'io.github.containers.compression.zstd' (io.github.containers.compression.zstd must be set)"
+    ;;
+  esac
+
+  run jq -r '.manifests['$1'].platform.architecture' <<< $2
+  assert "$output" = $3 ".manifests[$1].platform.architecture"
+}
+
 # Regression test for #8931
 @test "podman images - bare manifest list" {
     # Create an empty manifest list and list images.
@@ -56,4 +89,46 @@ load helpers.registry
     is "$output" ".*\"mediaType\": \"application/vnd.docker.distribution.manifest.list.v2+json\"" "Verify --tls-verify=false with REGISTRY_AUTH_FILE works against an insecure registry"
 }
 
+@test "manifest list --add-compression with zstd" {
+    if ! type -p skopeo; then
+        skip "skopeo not available"
+    fi
+    skip_if_remote "running a local registry doesn't work with podman-remote"
+    start_registry
+
+    tmpdir=$PODMAN_TMPDIR/build-test
+    mkdir -p $tmpdir
+    dockerfile=$tmpdir/Dockerfile
+    cat >$dockerfile <<EOF
+FROM alpine
+EOF
+    authfile=${PODMAN_LOGIN_WORKDIR}/auth-$(random_string 10).json
+    run_podman login --tls-verify=false \
+               --username ${PODMAN_LOGIN_USER} \
+               --password-stdin \
+               --authfile=$authfile \
+               localhost:${PODMAN_LOGIN_REGISTRY_PORT} <<<"${PODMAN_LOGIN_PASS}"
+    is "$output" "Login Succeeded!" "output from podman login"
+
+  manifest1="localhost:${PODMAN_LOGIN_REGISTRY_PORT}/test:1.0"
+  run_podman build -t image1 --platform linux/amd64 -f $dockerfile
+  run_podman build -t image2 --platform linux/arm64 -f $dockerfile
+
+  run_podman manifest create foo
+  run_podman images -a
+  run_podman manifest add foo containers-storage:localhost/image1:latest
+  run_podman manifest add foo containers-storage:localhost/image2:latest
+
+  run_podman manifest push --authfile=$authfile --all --add-compression zstd --tls-verify=false foo $manifest1
+
+  run skopeo inspect --authfile=$authfile --tls-verify=false --raw docker://$manifest1
+  echo $output
+  list="$output"
+
+  validate_instance_compression "0" "$list" "amd64" "gzip"
+  validate_instance_compression "1" "$list" "arm64" "gzip"
+  validate_instance_compression "2" "$list" "amd64" "zstd"
+  validate_instance_compression "3" "$list" "arm64" "zstd"
+}
+
 # vim: filetype=sh

vendor/github.com/containers/image/v5/copy/multiple.go

@@ -122,6 +122,10 @@ func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest.
 		if err != nil {
 			return res, fmt.Errorf("getting details for instance %s: %w", instanceDigest, err)
 		}
+		res = append(res, instanceCopy{
+			op:           instanceCopyCopy,
+			sourceDigest: instanceDigest,
+		})
 		platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform)
 		compressionList := compressionsByPlatform[platform]
 		for _, compressionVariant := range options.EnsureCompressionVariantsExist {
@@ -137,10 +141,6 @@ func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest.
 				compressionList.Add(compressionVariant.Algorithm.Name())
 			}
 		}
-		res = append(res, instanceCopy{
-			op:           instanceCopyCopy,
-			sourceDigest: instanceDigest,
-		})
 	}
 	return res, nil
 }

vendor/modules.txt

@@ -215,7 +215,7 @@ github.com/containers/common/version
 # github.com/containers/conmon v2.0.20+incompatible
 ## explicit
 github.com/containers/conmon/runner/config
-# github.com/containers/image/v5 v5.26.1-0.20230726142307-8c387a14f4ac
+# github.com/containers/image/v5 v5.26.1-0.20230801083106-fcf7f0e1712a
 ## explicit; go 1.18
 github.com/containers/image/v5/copy
 github.com/containers/image/v5/directory
@@ -399,7 +399,7 @@ github.com/crc-org/vfkit/pkg/config
 github.com/crc-org/vfkit/pkg/rest
 github.com/crc-org/vfkit/pkg/rest/define
 github.com/crc-org/vfkit/pkg/util
-# github.com/cyberphone/json-canonicalization v0.0.0-20230701045847-91eb5f1b7744
+# github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd
 ## explicit
 github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer
 # github.com/cyphar/filepath-securejoin v0.2.3