From 346f9cb4ed7cd6c6047680fc5f5223765539946d Mon Sep 17 00:00:00 2001 From: Aditya R Date: Tue, 1 Aug 2023 15:31:52 +0530 Subject: [PATCH] manifest/push: add support for --add-compression Adds support for --add-compression which accepts multiple compression formats and when used it will add all instances in a manifest list with requested compression formats. Signed-off-by: Aditya R --- cmd/podman/manifest/push.go | 4 + .../markdown/podman-manifest-push.1.md.in | 11 +++ pkg/api/handlers/libpod/manifests.go | 16 ++-- pkg/api/server/register_manifest.go | 7 ++ pkg/bindings/images/types.go | 2 + pkg/bindings/images/types_push_options.go | 15 ++++ pkg/domain/entities/images.go | 3 + pkg/domain/infra/abi/manifest.go | 1 + pkg/domain/infra/tunnel/manifest.go | 2 +- test/e2e/manifest_test.go | 77 +++++++++++++++++++ test/system/012-manifest.bats | 75 ++++++++++++++++++ 11 files changed, 205 insertions(+), 8 deletions(-) diff --git a/cmd/podman/manifest/push.go b/cmd/podman/manifest/push.go index bf25951b44..967bfbbd83 100644 --- a/cmd/podman/manifest/push.go +++ b/cmd/podman/manifest/push.go @@ -56,6 +56,10 @@ func init() { flags.StringVar(&manifestPushOpts.Authfile, authfileFlagName, auth.GetDefaultAuthFile(), "path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override") _ = pushCmd.RegisterFlagCompletionFunc(authfileFlagName, completion.AutocompleteDefault) + addCompressionFlagName := "add-compression" + flags.StringSliceVar(&manifestPushOpts.AddCompression, addCompressionFlagName, nil, "add instances with selected compression while pushing") + _ = pushCmd.RegisterFlagCompletionFunc(addCompressionFlagName, common.AutocompleteCompressionFormat) + certDirFlagName := "cert-dir" flags.StringVar(&manifestPushOpts.CertDir, certDirFlagName, "", "use certificates at the specified path to access the registry") _ = pushCmd.RegisterFlagCompletionFunc(certDirFlagName, completion.AutocompleteDefault) diff --git a/docs/source/markdown/podman-manifest-push.1.md.in b/docs/source/markdown/podman-manifest-push.1.md.in index fd2f365d4b..fcdcde66e5 100644 --- a/docs/source/markdown/podman-manifest-push.1.md.in +++ b/docs/source/markdown/podman-manifest-push.1.md.in @@ -14,6 +14,17 @@ The list image's ID and the digest of the image's manifest. ## OPTIONS +#### **--add-compression**=*compression* + +Makes sure that requested compression variant for each platform is added to the manifest list keeping original instance +intact in the same manifest list. Supported values are (`gzip`, `zstd` and `zstd:chunked`). Following flag can be used +multiple times. + +Note that `--compression-format` controls the compression format of each instance in the manifest list. `--add-compression` +will add another variant for each instance in the list with the specified compressions. `--compression-format` gzip `--add-compression` +zstd will push a manifest list with each instance being compressed with gzip plus an additional variant of each instance +being compressed with zstd. + #### **--all** Push the images mentioned in the manifest list or image index, in addition to diff --git a/pkg/api/handlers/libpod/manifests.go b/pkg/api/handlers/libpod/manifests.go index b94b937b97..0434d5fbc0 100644 --- a/pkg/api/handlers/libpod/manifests.go +++ b/pkg/api/handlers/libpod/manifests.go @@ -333,13 +333,14 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) query := struct { - All bool `schema:"all"` - CompressionFormat string `schema:"compressionFormat"` - CompressionLevel *int `schema:"compressionLevel"` - Format string `schema:"format"` - RemoveSignatures bool `schema:"removeSignatures"` - TLSVerify bool `schema:"tlsVerify"` - Quiet bool `schema:"quiet"` + All bool `schema:"all"` + CompressionFormat string `schema:"compressionFormat"` + CompressionLevel *int `schema:"compressionLevel"` + Format string `schema:"format"` + RemoveSignatures bool `schema:"removeSignatures"` + TLSVerify bool `schema:"tlsVerify"` + Quiet bool `schema:"quiet"` + AddCompression []string `schema:"addCompression"` }{ // Add defaults here once needed. TLSVerify: true, @@ -373,6 +374,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { options := entities.ImagePushOptions{ All: query.All, Authfile: authfile, + AddCompression: query.AddCompression, CompressionFormat: query.CompressionFormat, CompressionLevel: query.CompressionLevel, Format: query.Format, diff --git a/pkg/api/server/register_manifest.go b/pkg/api/server/register_manifest.go index 90edad1d02..282bcd8d85 100644 --- a/pkg/api/server/register_manifest.go +++ b/pkg/api/server/register_manifest.go @@ -60,6 +60,13 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // type: string // required: true // description: the name or ID of the manifest list + // - in: query + // name: addCompression + // required: false + // description: add existing instances with requested compression algorithms to manifest list + // type: array + // items: + // type: string // - in: path // name: destination // type: string diff --git a/pkg/bindings/images/types.go b/pkg/bindings/images/types.go index 9c3ed2b108..e8ac86ce5b 100644 --- a/pkg/bindings/images/types.go +++ b/pkg/bindings/images/types.go @@ -144,6 +144,8 @@ type PushOptions struct { CompressionFormat *string // CompressionLevel is the level to use for the compression of the blobs CompressionLevel *int + // Add existing instances with requested compression algorithms to manifest list + AddCompression []string // Manifest type of the pushed image Format *string // Password for authenticating against the registry. diff --git a/pkg/bindings/images/types_push_options.go b/pkg/bindings/images/types_push_options.go index 550b1f4070..7d5d38e1c9 100644 --- a/pkg/bindings/images/types_push_options.go +++ b/pkg/bindings/images/types_push_options.go @@ -93,6 +93,21 @@ func (o *PushOptions) GetCompressionLevel() int { return *o.CompressionLevel } +// WithAddCompression set field AddCompression to given value +func (o *PushOptions) WithAddCompression(value []string) *PushOptions { + o.AddCompression = value + return o +} + +// GetAddCompression returns value of field AddCompression +func (o *PushOptions) GetAddCompression() []string { + if o.AddCompression == nil { + var z []string + return z + } + return o.AddCompression +} + // WithFormat set field Format to given value func (o *PushOptions) WithFormat(value string) *PushOptions { o.Format = &value diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index c58594d983..cd2062c928 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -244,6 +244,9 @@ type ImagePushOptions struct { // integers in the slice represent 0-indexed layer indices, with support for negative // indexing. i.e. 0 is the first layer, -1 is the last (top-most) layer. OciEncryptLayers *[]int + // If necessary, add clones of existing instances with requested compression algorithms to manifest list + // Note: Following option is only valid for `manifest push` + AddCompression []string } // ImagePushReport is the response from pushing an image. diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index 224c8488b9..4acf09038a 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -345,6 +345,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin pushOptions.InsecureSkipTLSVerify = opts.SkipTLSVerify pushOptions.Writer = opts.Writer pushOptions.CompressionLevel = opts.CompressionLevel + pushOptions.AddCompression = opts.AddCompression compressionFormat := opts.CompressionFormat if compressionFormat == "" { diff --git a/pkg/domain/infra/tunnel/manifest.go b/pkg/domain/infra/tunnel/manifest.go index ca708ab0b0..d2bd4e762e 100644 --- a/pkg/domain/infra/tunnel/manifest.go +++ b/pkg/domain/infra/tunnel/manifest.go @@ -135,7 +135,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin } options := new(images.PushOptions) - options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer) + options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer).WithAddCompression(opts.AddCompression) if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined { if s == types.OptionalBoolTrue { diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go index 14a5abe09c..cba3029691 100644 --- a/test/e2e/manifest_test.go +++ b/test/e2e/manifest_test.go @@ -13,8 +13,28 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gexec" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) +// Internal function to verify instance compression +func verifyInstanceCompression(descriptor []imgspecv1.Descriptor, compression string, arch string) bool { + for _, instance := range descriptor { + if instance.Platform.Architecture != arch { + continue + } + if compression == "zstd" { + // if compression is zstd annotations must contain + val, ok := instance.Annotations["io.github.containers.compression.zstd"] + if ok && val == "true" { + return true + } + } else if len(instance.Annotations) == 0 { + return true + } + } + return false +} + var _ = Describe("Podman manifest", func() { const ( @@ -135,6 +155,63 @@ var _ = Describe("Podman manifest", func() { Expect(session2.OutputToString()).To(Equal(session.OutputToString())) }) + It("push with --add-compression", func() { + if podmanTest.Host.Arch == "ppc64le" { + Skip("No registry image for ppc64le") + } + if isRootless() { + err := podmanTest.RestoreArtifact(REGISTRY_IMAGE) + Expect(err).ToNot(HaveOccurred()) + } + lock := GetPortLock("5000") + defer lock.Unlock() + session := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", REGISTRY_IMAGE, "/entrypoint.sh", "/etc/docker/registry/config.yml"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + if !WaitContainerReady(podmanTest, "registry", "listening on", 20, 1) { + Skip("Cannot start docker registry.") + } + + session = podmanTest.Podman([]string{"build", "--platform", "linux/amd64", "-t", "imageone", "build/basicalpine"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"build", "--platform", "linux/arm64", "-t", "imagetwo", "build/basicalpine"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"manifest", "create", "foobar"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + session = podmanTest.Podman([]string{"manifest", "add", "foobar", "containers-storage:localhost/imageone:latest"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + session = podmanTest.Podman([]string{"manifest", "add", "foobar", "containers-storage:localhost/imagetwo:latest"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + push := podmanTest.Podman([]string{"manifest", "push", "--all", "--add-compression", "zstd", "--tls-verify=false", "--remove-signatures", "foobar", "localhost:5000/list"}) + push.WaitWithDefaultTimeout() + Expect(push).Should(Exit(0)) + output := push.ErrorToString() + // 4 images must be pushed two for gzip and two for zstd + Expect(output).To(ContainSubstring("Copying 4 images generated from 2 images in list")) + + session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/list:latest"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + var index imgspecv1.Index + inspectData := []byte(session.OutputToString()) + err := json.Unmarshal(inspectData, &index) + Expect(err).ToNot(HaveOccurred()) + + Expect(verifyInstanceCompression(index.Manifests, "zstd", "amd64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeTrue()) + }) + It("add --all", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() diff --git a/test/system/012-manifest.bats b/test/system/012-manifest.bats index df90a135f2..6f99454a85 100644 --- a/test/system/012-manifest.bats +++ b/test/system/012-manifest.bats @@ -4,6 +4,39 @@ load helpers load helpers.network load helpers.registry +# Helper function for several of the tests which verifies compression. +# +# Usage: validate_instance_compression INDEX MANIFEST ARCH COMPRESSION +# +# INDEX instance which needs to be verified in +# provided manifest list. +# +# MANIFEST OCI manifest specification in json format +# +# ARCH instance architecture +# +# COMPRESSION compression algorithm name; e.g "zstd". +# +function validate_instance_compression { + case $4 in + + gzip) + run jq -r '.manifests['$1'].annotations' <<< $2 + # annotation is `null` for gzip compression + assert "$output" = "null" ".manifests[$1].annotations (null means gzip)" + ;; + + zstd) + # annotation `'"io.github.containers.compression.zstd": "true"'` must be there for zstd compression + run jq -r '.manifests['$1'].annotations."io.github.containers.compression.zstd"' <<< $2 + assert "$output" = "true" ".manifests[$1].annotations.'io.github.containers.compression.zstd' (io.github.containers.compression.zstd must be set)" + ;; + esac + + run jq -r '.manifests['$1'].platform.architecture' <<< $2 + assert "$output" = $3 ".manifests[$1].platform.architecture" +} + # Regression test for #8931 @test "podman images - bare manifest list" { # Create an empty manifest list and list images. @@ -56,4 +89,46 @@ load helpers.registry is "$output" ".*\"mediaType\": \"application/vnd.docker.distribution.manifest.list.v2+json\"" "Verify --tls-verify=false with REGISTRY_AUTH_FILE works against an insecure registry" } +@test "manifest list --add-compression with zstd" { + if ! type -p skopeo; then + skip "skopeo not available" + fi + skip_if_remote "running a local registry doesn't work with podman-remote" + start_registry + + tmpdir=$PODMAN_TMPDIR/build-test + mkdir -p $tmpdir + dockerfile=$tmpdir/Dockerfile + cat >$dockerfile <