From 82bd56be74d6f2b79ee7eb3fe2923562cad47a14 Mon Sep 17 00:00:00 2001 From: Aditya R Date: Wed, 16 Aug 2023 12:49:11 +0530 Subject: [PATCH] manifest-push: add support for --force-compression Adds support for --force-compression which allows end-users to force push blobs with the selected compresison in --compression option, in order to make sure that blobs of other compression on registry are not reused. Signed-off-by: Aditya R --- cmd/podman/manifest/push.go | 2 + .../markdown/podman-manifest-push.1.md.in | 2 + docs/source/markdown/podman-push.1.md.in | 4 +- pkg/api/handlers/libpod/manifests.go | 38 ++++++++-------- pkg/api/server/register_manifest.go | 5 +++ pkg/domain/infra/abi/manifest.go | 1 + pkg/domain/infra/tunnel/manifest.go | 2 +- test/e2e/manifest_test.go | 45 ++++++++++++++++++- 8 files changed, 78 insertions(+), 21 deletions(-) diff --git a/cmd/podman/manifest/push.go b/cmd/podman/manifest/push.go index 967bfbbd83..4bc8c6159c 100644 --- a/cmd/podman/manifest/push.go +++ b/cmd/podman/manifest/push.go @@ -72,6 +72,8 @@ func init() { flags.StringVar(&manifestPushOpts.DigestFile, digestfileFlagName, "", "after copying the image, write the digest of the resulting digest to the file") _ = pushCmd.RegisterFlagCompletionFunc(digestfileFlagName, completion.AutocompleteDefault) + flags.BoolVar(&manifestPushOpts.ForceCompressionFormat, "force-compression", false, "Use the specified compression algorithm if the destination contains a differently-compressed variant already") + formatFlagName := "format" flags.StringVarP(&manifestPushOpts.Format, formatFlagName, "f", "", "manifest type (oci or v2s2) to attempt to use when pushing the manifest list (default is manifest type of source)") _ = pushCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteManifestFormat) diff --git a/docs/source/markdown/podman-manifest-push.1.md.in b/docs/source/markdown/podman-manifest-push.1.md.in index 84ea14ca75..4bfeccc83b 100644 --- a/docs/source/markdown/podman-manifest-push.1.md.in +++ b/docs/source/markdown/podman-manifest-push.1.md.in @@ -45,6 +45,8 @@ the list or index itself. (Default true) #### **--force-compression** Use the specified compression algorithm even if the destination contains a differently-compressed variant already. +Usually use for this flag arises when image is prior compressed and pushed using `--compression-format` with a different +compression algorithm and user now needs to overwrite those blobs with a new compression algorithm on the remote registry. #### **--format**, **-f**=*format* diff --git a/docs/source/markdown/podman-push.1.md.in b/docs/source/markdown/podman-push.1.md.in index b450bf9829..71a0c79052 100644 --- a/docs/source/markdown/podman-push.1.md.in +++ b/docs/source/markdown/podman-push.1.md.in @@ -70,9 +70,11 @@ Layer(s) to encrypt: 0-indexed layer indices with support for negative indexing The [protocol:keyfile] specifies the encryption protocol, which can be JWE (RFC7516), PGP (RFC4880), and PKCS7 (RFC2315) and the key material required for image encryption. For instance, jwe:/path/to/key.pem or pgp:admin@example.com or pkcs7:/path/to/x509-file. -### **--force-compression** +#### **--force-compression** Use the specified compression algorithm even if the destination contains a differently-compressed variant already. +Usually use for this flag arises when image is prior compressed and pushed using `--compression-format` with a different +compression algorithm and user now needs to overwrite those blobs with a new compression algorithm on the remote registry. #### **--format**, **-f**=*format* diff --git a/pkg/api/handlers/libpod/manifests.go b/pkg/api/handlers/libpod/manifests.go index 0434d5fbc0..5094796e43 100644 --- a/pkg/api/handlers/libpod/manifests.go +++ b/pkg/api/handlers/libpod/manifests.go @@ -333,14 +333,15 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) query := struct { - All bool `schema:"all"` - CompressionFormat string `schema:"compressionFormat"` - CompressionLevel *int `schema:"compressionLevel"` - Format string `schema:"format"` - RemoveSignatures bool `schema:"removeSignatures"` - TLSVerify bool `schema:"tlsVerify"` - Quiet bool `schema:"quiet"` - AddCompression []string `schema:"addCompression"` + All bool `schema:"all"` + CompressionFormat string `schema:"compressionFormat"` + CompressionLevel *int `schema:"compressionLevel"` + ForceCompressionFormat bool `schema:"forceCompressionFormat"` + Format string `schema:"format"` + RemoveSignatures bool `schema:"removeSignatures"` + TLSVerify bool `schema:"tlsVerify"` + Quiet bool `schema:"quiet"` + AddCompression []string `schema:"addCompression"` }{ // Add defaults here once needed. TLSVerify: true, @@ -372,16 +373,17 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { password = authconf.Password } options := entities.ImagePushOptions{ - All: query.All, - Authfile: authfile, - AddCompression: query.AddCompression, - CompressionFormat: query.CompressionFormat, - CompressionLevel: query.CompressionLevel, - Format: query.Format, - Password: password, - Quiet: true, - RemoveSignatures: query.RemoveSignatures, - Username: username, + All: query.All, + Authfile: authfile, + AddCompression: query.AddCompression, + CompressionFormat: query.CompressionFormat, + CompressionLevel: query.CompressionLevel, + ForceCompressionFormat: query.ForceCompressionFormat, + Format: query.Format, + Password: password, + Quiet: true, + RemoveSignatures: query.RemoveSignatures, + Username: username, } if sys := runtime.SystemContext(); sys != nil { options.CertDir = sys.DockerCertPath diff --git a/pkg/api/server/register_manifest.go b/pkg/api/server/register_manifest.go index 282bcd8d85..624dcd4afe 100644 --- a/pkg/api/server/register_manifest.go +++ b/pkg/api/server/register_manifest.go @@ -67,6 +67,11 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // type: array // items: // type: string + // - in: query + // name: forceCompressionFormat + // description: Use the specified compression algorithm if the destination contains a differently-compressed variant already. + // type: boolean + // default: false // - in: path // name: destination // type: string diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index 4acf09038a..b664f6678a 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -346,6 +346,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin pushOptions.Writer = opts.Writer pushOptions.CompressionLevel = opts.CompressionLevel pushOptions.AddCompression = opts.AddCompression + pushOptions.ForceCompressionFormat = opts.ForceCompressionFormat compressionFormat := opts.CompressionFormat if compressionFormat == "" { diff --git a/pkg/domain/infra/tunnel/manifest.go b/pkg/domain/infra/tunnel/manifest.go index d2bd4e762e..d1cb0274a1 100644 --- a/pkg/domain/infra/tunnel/manifest.go +++ b/pkg/domain/infra/tunnel/manifest.go @@ -135,7 +135,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin } options := new(images.PushOptions) - options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer).WithAddCompression(opts.AddCompression) + options.WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithRemoveSignatures(opts.RemoveSignatures).WithAll(opts.All).WithFormat(opts.Format).WithCompressionFormat(opts.CompressionFormat).WithQuiet(opts.Quiet).WithProgressWriter(opts.Writer).WithAddCompression(opts.AddCompression).WithForceCompressionFormat(opts.ForceCompressionFormat) if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined { if s == types.OptionalBoolTrue { diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go index 52ba578ba3..73d2978e28 100644 --- a/test/e2e/manifest_test.go +++ b/test/e2e/manifest_test.go @@ -154,7 +154,7 @@ var _ = Describe("Podman manifest", func() { Expect(session2.OutputToString()).To(Equal(session.OutputToString())) }) - It("push with --add-compression", func() { + It("push with --add-compression and --force-compression", func() { if podmanTest.Host.Arch == "ppc64le" { Skip("No registry image for ppc64le") } @@ -209,6 +209,49 @@ var _ = Describe("Podman manifest", func() { Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue()) Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeTrue()) Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeTrue()) + + // Note: Pushing again with --force-compression should produce the correct response the since blobs will be correctly force-pushed again. + push = podmanTest.Podman([]string{"manifest", "push", "--all", "--add-compression", "zstd", "--tls-verify=false", "--compression-format", "gzip", "--force-compression", "--remove-signatures", "foobar", "localhost:5000/list"}) + push.WaitWithDefaultTimeout() + Expect(push).Should(Exit(0)) + output = push.ErrorToString() + // 4 images must be pushed two for gzip and two for zstd + Expect(output).To(ContainSubstring("Copying 4 images generated from 2 images in list")) + + session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/list:latest"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + inspectData = []byte(session.OutputToString()) + err = json.Unmarshal(inspectData, &index) + Expect(err).ToNot(HaveOccurred()) + + Expect(verifyInstanceCompression(index.Manifests, "zstd", "amd64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeTrue()) + + // Note: Pushing again without --force-compression should produce in-correct/wrong result since blobs are already present in registry so they will be reused + // ignoring our compression priority ( this is expected behaviour of c/image and --force-compression is introduced to mitigate this behaviour ). + push = podmanTest.Podman([]string{"manifest", "push", "--all", "--add-compression", "zstd", "--tls-verify=false", "--remove-signatures", "foobar", "localhost:5000/list"}) + push.WaitWithDefaultTimeout() + Expect(push).Should(Exit(0)) + output = push.ErrorToString() + // 4 images must be pushed two for gzip and two for zstd + Expect(output).To(ContainSubstring("Copying 4 images generated from 2 images in list")) + + session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/list:latest"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + inspectData = []byte(session.OutputToString()) + err = json.Unmarshal(inspectData, &index) + Expect(err).ToNot(HaveOccurred()) + + Expect(verifyInstanceCompression(index.Manifests, "zstd", "amd64")).Should(BeTrue()) + Expect(verifyInstanceCompression(index.Manifests, "zstd", "arm64")).Should(BeTrue()) + // blobs of zstd will be wrongly reused for gzip instances without --force-compression + Expect(verifyInstanceCompression(index.Manifests, "gzip", "arm64")).Should(BeFalse()) + // blobs of zstd will be wrongly reused for gzip instances without --force-compression + Expect(verifyInstanceCompression(index.Manifests, "gzip", "amd64")).Should(BeFalse()) }) It("add --all", func() {