Allow podman pull to specify --retry and --retry-delay

Fixes: https://github.com/containers/podman/issues/19770

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

cmd/podman/images/pull.go

@@ -113,6 +113,13 @@ func pullFlags(cmd *cobra.Command) {
 	flags.StringArrayVar(&pullOptions.DecryptionKeys, decryptionKeysFlagName, nil, "Key needed to decrypt the image (e.g. /path/to/key.pem)")
 	_ = cmd.RegisterFlagCompletionFunc(decryptionKeysFlagName, completion.AutocompleteDefault)
 
+	retryFlagName := "retry"
+	flags.Uint(retryFlagName, cli.MaxPullPushRetries, "number of times to retry in case of failure when performing pull")
+	_ = cmd.RegisterFlagCompletionFunc(retryFlagName, completion.AutocompleteNone)
+	retryDelayFlagName := "retry-delay"
+	flags.String(retryDelayFlagName, cli.PullPushRetryDelay.String(), "delay between retries in case of pull failures")
+	_ = cmd.RegisterFlagCompletionFunc(retryDelayFlagName, completion.AutocompleteNone)
+
 	if registry.IsRemote() {
 		_ = flags.MarkHidden(decryptionKeysFlagName)
 	}
@@ -136,6 +143,25 @@ func imagePull(cmd *cobra.Command, args []string) error {
 	if cmd.Flags().Changed("tls-verify") {
 		pullOptions.SkipTLSVerify = types.NewOptionalBool(!pullOptions.TLSVerifyCLI)
 	}
+
+	if cmd.Flags().Changed("retry") {
+		retry, err := cmd.Flags().GetUint("retry")
+		if err != nil {
+			return err
+		}
+
+		pullOptions.Retry = &retry
+	}
+
+	if cmd.Flags().Changed("retry-delay") {
+		val, err := cmd.Flags().GetString("retry-delay")
+		if err != nil {
+			return err
+		}
+
+		pullOptions.RetryDelay = val
+	}
+
 	if cmd.Flags().Changed("authfile") {
 		if err := auth.CheckAuthFile(pullOptions.Authfile); err != nil {
 			return err

docs/source/markdown/options/retry-delay.md

@@ -1,5 +1,5 @@
 ####> This option file is used in:
-####>   podman build, farm build
+####>   podman build, farm build, pull
 ####> If file is edited, make sure the changes
 ####> are applicable to all of those.
 #### **--retry-delay**=*duration*

docs/source/markdown/options/retry.md

@@ -1,5 +1,5 @@
 ####> This option file is used in:
-####>   podman build, farm build
+####>   podman build, farm build, pull
 ####> If file is edited, make sure the changes
 ####> are applicable to all of those.
 #### **--retry**=*attempts*

docs/source/markdown/podman-pull.1.md.in

@@ -73,6 +73,10 @@ Print the usage statement.
 
 Suppress output information when pulling images
 
+@@option retry
+
+@@option retry-delay
+
 @@option tls-verify
 
 @@option variant.container
@@ -205,6 +209,10 @@ Storing signatures
 3cba58dad5d9b35e755b48b634acb3fdd185ab1c996ac11510cc72c17780e13c
 ```
 
+Pull an image with up to 6 retries, delaying 10 seconds between retries in quet mode.
+$ podman --remote pull -q --retry 6 --retry-delay 10s ubi9
+4d6addf62a90e392ff6d3f470259eb5667eab5b9a8e03d20b41d0ab910f92170
+
 ## SEE ALSO
 **[podman(1)](podman.1.md)**, **[podman-push(1)](podman-push.1.md)**, **[podman-login(1)](podman-login.1.md)**, **[containers-certs.d(5)](https://github.com/containers/image/blob/main/docs/containers-certs.d.5.md)**, **[containers-registries.conf(5)](https://github.com/containers/image/blob/main/docs/containers-registries.conf.5.md)**, **[containers-transports(5)](https://github.com/containers/image/blob/main/docs/containers-transports.5.md)**
 

pkg/api/handlers/compat/images.go

@@ -260,6 +260,8 @@ func CreateImageFromImage(w http.ResponseWriter, r *http.Request) {
 		FromImage  string `schema:"fromImage"`
 		Tag        string `schema:"tag"`
 		Platform   string `schema:"platform"`
+		Retry      uint   `schema:"retry"`
+		RetryDelay string `schema:"retryDelay"`
 	}{
 		// This is where you can override the golang default value for one of fields
 	}
@@ -290,6 +292,19 @@ func CreateImageFromImage(w http.ResponseWriter, r *http.Request) {
 	}
 	pullOptions.Writer = os.Stderr // allows for debugging on the server
 
+	if _, found := r.URL.Query()["retry"]; found {
+		pullOptions.MaxRetries = &query.Retry
+	}
+
+	if _, found := r.URL.Query()["retryDelay"]; found {
+		duration, err := time.ParseDuration(query.RetryDelay)
+		if err != nil {
+			utils.Error(w, http.StatusBadRequest, err)
+			return
+		}
+		pullOptions.RetryDelay = &duration
+	}
+
 	// Handle the platform.
 	platformSpecs := strings.Split(query.Platform, "/")
 	pullOptions.OS = platformSpecs[0] // may be empty

pkg/api/handlers/libpod/images_pull.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/containers/common/libimage"
 	"github.com/containers/common/pkg/config"
@@ -33,6 +34,8 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) {
 		PullPolicy string `schema:"policy"`
 		Quiet      bool   `schema:"quiet"`
 		Reference  string `schema:"reference"`
+		Retry      uint   `schema:"retry"`
+		RetryDelay string `schema:"retrydelay"`
 		TLSVerify  bool   `schema:"tlsVerify"`
 		// Platform fields below:
 		Arch    string `schema:"Arch"`
@@ -95,6 +98,19 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if _, found := r.URL.Query()["retry"]; found {
+		pullOptions.MaxRetries = &query.Retry
+	}
+
+	if _, found := r.URL.Query()["retrydelay"]; found {
+		duration, err := time.ParseDuration(query.RetryDelay)
+		if err != nil {
+			utils.Error(w, http.StatusBadRequest, err)
+			return
+		}
+		pullOptions.RetryDelay = &duration
+	}
+
 	// Let's keep thing simple when running in quiet mode and pull directly.
 	if query.Quiet {
 		images, err := runtime.LibimageRuntime().Pull(r.Context(), query.Reference, pullPolicy, pullOptions)

pkg/bindings/images/types.go

@@ -217,6 +217,10 @@ type PullOptions struct {
 	// Quiet can be specified to suppress pull progress when pulling.  Ignored
 	// for remote calls.
 	Quiet *bool
+	// Retry number of times to retry pull in case of failure
+	Retry *uint
+	// RetryDelay between retries in case of pull failures
+	RetryDelay *string
 	// SkipTLSVerify to skip HTTPS and certificate verification.
 	SkipTLSVerify *bool `schema:"-"`
 	// Username for authenticating against the registry.

pkg/bindings/images/types_pull_options.go

@@ -138,6 +138,36 @@ func (o *PullOptions) GetQuiet() bool {
 	return *o.Quiet
 }
 
+// WithRetry set field Retry to given value
+func (o *PullOptions) WithRetry(value uint) *PullOptions {
+	o.Retry = &value
+	return o
+}
+
+// GetRetry returns value of field Retry
+func (o *PullOptions) GetRetry() uint {
+	if o.Retry == nil {
+		var z uint
+		return z
+	}
+	return *o.Retry
+}
+
+// WithRetryDelay set field RetryDelay to given value
+func (o *PullOptions) WithRetryDelay(value string) *PullOptions {
+	o.RetryDelay = &value
+	return o
+}
+
+// GetRetryDelay returns value of field RetryDelay
+func (o *PullOptions) GetRetryDelay() string {
+	if o.RetryDelay == nil {
+		var z string
+		return z
+	}
+	return *o.RetryDelay
+}
+
 // WithSkipTLSVerify set field SkipTLSVerify to given value
 func (o *PullOptions) WithSkipTLSVerify(value bool) *PullOptions {
 	o.SkipTLSVerify = &value

pkg/domain/entities/images.go

@@ -102,6 +102,10 @@ type ImagePullOptions struct {
 	// Quiet can be specified to suppress pull progress when pulling.  Ignored
 	// for remote calls.
 	Quiet bool
+	// Retry number of times to retry pull in case of failure
+	Retry *uint
+	// RetryDelay between retries in case of pull failures
+	RetryDelay string
 	// SignaturePolicy to use when pulling.  Ignored for remote calls.
 	SignaturePolicy string
 	// SkipTLSVerify to skip HTTPS and certificate verification.

pkg/domain/infra/abi/images.go

@@ -14,6 +14,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	bdefine "github.com/containers/buildah/define"
 	"github.com/containers/common/libimage"
@@ -254,6 +255,15 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, options entiti
 	pullOptions.Writer = options.Writer
 	pullOptions.OciDecryptConfig = options.OciDecryptConfig
 
+	pullOptions.MaxRetries = options.Retry
+	if options.RetryDelay != "" {
+		duration, err := time.ParseDuration(options.RetryDelay)
+		if err != nil {
+			return nil, err
+		}
+		pullOptions.RetryDelay = &duration
+	}
+
 	if !options.Quiet && pullOptions.Writer == nil {
 		pullOptions.Writer = os.Stderr
 	}

pkg/domain/infra/tunnel/images.go

@@ -123,6 +123,12 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, opts entities.
 			options.WithSkipTLSVerify(false)
 		}
 	}
+	if opts.Retry != nil {
+		options.WithRetry(*opts.Retry)
+	}
+	if opts.RetryDelay != "" {
+		options.WithRetryDelay(opts.RetryDelay)
+	}
 	pulledImages, err := images.Pull(ir.ClientCtx, rawImage, options)
 	if err != nil {
 		return nil, err

test/system/150-login.bats

@@ -327,6 +327,46 @@ function _test_skopeo_credential_sharing() {
 
 }
 
+@test "podman images with retry" {
+    run_podman pull -q --retry 4 --retry-delay "10s" $IMAGE
+    run_podman 125 pull -q --retry 4 --retry-delay "bogus" $IMAGE
+    is "$output" 'Error: time: invalid duration "bogus"' "bad retry-delay"
+
+    skip_if_remote "running a local registry doesn't work with podman-remote"
+    start_registry
+    authfile=${PODMAN_LOGIN_WORKDIR}/auth-$(random_string 10).json
+    run_podman login --tls-verify=false \
+               --username ${PODMAN_LOGIN_USER} \
+               --password-stdin \
+               --authfile=$authfile \
+               localhost:${PODMAN_LOGIN_REGISTRY_PORT} <<<"${PODMAN_LOGIN_PASS}"
+    is "$output" "Login Succeeded!" "output from podman login"
+
+    image1="localhost:${PODMAN_LOGIN_REGISTRY_PORT}/test:1.0"
+
+    run_podman tag $IMAGE $image1
+    run_podman push --authfile=$authfile \
+        --tls-verify=false $mid \
+        $image1
+    run_podman rmi $image1
+    run_podman pull -q --retry 4 --retry-delay "0s" --authfile=$authfile \
+        --tls-verify=false $image1
+    assert "${output:0:12}" = "$PODMAN_TEST_IMAGE_ID" "First pull (before stopping registry)"
+    run_podman rmi $image1
+
+    # This actually STOPs the registry, so the port is unbound...
+    pause_registry
+    # ...then, in eight seconds, we start it again
+    (sleep 8; unpause_registry) &
+    run_podman 0+w pull -q --retry 4 --retry-delay "5s" --authfile=$authfile \
+            --tls-verify=false $image1
+    assert "$output" =~ "Failed, retrying in 5s.*Error: initializing.* connection refused"
+    assert "${lines[-1]:0:12}" = "$PODMAN_TEST_IMAGE_ID" "push should succeed via retry"
+    unpause_registry
+
+    run_podman rmi $image1
+}
+
 # END   cooperation with skopeo
 # END   actual tests
 ###############################################################################

test/system/helpers.registry.bash

@@ -115,3 +115,23 @@ function stop_registry() {
         die "Socket still seems open"
     fi
 }
+
+function pause_registry() {
+    if [[ ! -d "$PODMAN_LOGIN_WORKDIR/auth" ]]; then
+        # No registry running
+        return
+    fi
+
+    opts="--storage-driver vfs $(podman_isolation_opts ${PODMAN_LOGIN_WORKDIR})"
+    run_podman $opts stop registry
+}
+
+function unpause_registry() {
+    if [[ ! -d "$PODMAN_LOGIN_WORKDIR/auth" ]]; then
+        # No registry running
+        return
+    fi
+
+    opts="--storage-driver vfs $(podman_isolation_opts ${PODMAN_LOGIN_WORKDIR})"
+    run_podman $opts start registry
+}