podmanV2: implement pull

Implement pulling images for the v2 client.  What I _really_ don't like
is the fact that we are now having a near identical code clone among
`pkg/domain/infra/abi` and `pkg/api/handlers/libpod`.  Partly because we
don't yet have a higher-level pull function and partly because we have
redudancy among `pkg/domain` and `pkg/api`.  Pull might be a high
outlier but I am concerned already by the potential of introducing more
redundancy.  I'd love to `infra/abi` and `pkg/abi` to really use the
same code in the future.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

cmd/podmanV2/images/pull.go

@@ -0,0 +1,140 @@
+package images
+
+import (
+	"fmt"
+
+	buildahcli "github.com/containers/buildah/pkg/cli"
+	"github.com/containers/image/v5/types"
+	"github.com/containers/libpod/cmd/podmanV2/registry"
+	"github.com/containers/libpod/pkg/domain/entities"
+	"github.com/opentracing/opentracing-go"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+// pullOptionsWrapper wraps entities.ImagePullOptions and prevents leaking
+// CLI-only fields into the API types.
+type pullOptionsWrapper struct {
+	entities.ImagePullOptions
+	TLSVerifyCLI bool // CLI only
+}
+
+var (
+	pullOptions     = pullOptionsWrapper{}
+	pullDescription = `Pulls an image from a registry and stores it locally.
+
+  An image can be pulled by tag or digest. If a tag is not specified, the image with the 'latest' tag is pulled.`
+
+	// Command: podman pull
+	pullCmd = &cobra.Command{
+		Use:     "pull [flags] IMAGE",
+		Short:   "Pull an image from a registry",
+		Long:    pullDescription,
+		PreRunE: preRunE,
+		RunE:    imagePull,
+		Example: `podman pull imageName
+  podman pull fedora:latest`,
+	}
+
+	// Command: podman image pull
+	// It's basically a clone of `pullCmd` with the exception of being a
+	// child of the images command.
+	imagesPullCmd = &cobra.Command{
+		Use:     pullCmd.Use,
+		Short:   pullCmd.Short,
+		Long:    pullCmd.Long,
+		PreRunE: pullCmd.PreRunE,
+		RunE:    pullCmd.RunE,
+		Example: `podman image pull imageName
+  podman image pull fedora:latest`,
+	}
+)
+
+func init() {
+	// pull
+	registry.Commands = append(registry.Commands, registry.CliCommand{
+		Mode:    []entities.EngineMode{entities.ABIMode, entities.TunnelMode},
+		Command: pullCmd,
+	})
+
+	pullCmd.SetHelpTemplate(registry.HelpTemplate())
+	pullCmd.SetUsageTemplate(registry.UsageTemplate())
+
+	flags := pullCmd.Flags()
+	pullFlags(flags)
+
+	// images pull
+	registry.Commands = append(registry.Commands, registry.CliCommand{
+		Mode:    []entities.EngineMode{entities.ABIMode, entities.TunnelMode},
+		Command: imagesPullCmd,
+		Parent:  imageCmd,
+	})
+
+	imagesPullCmd.SetHelpTemplate(registry.HelpTemplate())
+	imagesPullCmd.SetUsageTemplate(registry.UsageTemplate())
+	imagesPullFlags := imagesPullCmd.Flags()
+	pullFlags(imagesPullFlags)
+}
+
+// pullFlags set the flags for the pull command.
+func pullFlags(flags *pflag.FlagSet) {
+	flags.BoolVar(&pullOptions.AllTags, "all-tags", false, "All tagged images in the repository will be pulled")
+	flags.StringVar(&pullOptions.Authfile, "authfile", buildahcli.GetDefaultAuthFile(), "Path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override")
+	flags.StringVar(&pullOptions.CertDir, "cert-dir", "", "`Pathname` of a directory containing TLS certificates and keys")
+	flags.StringVar(&pullOptions.Credentials, "creds", "", "`Credentials` (USERNAME:PASSWORD) to use for authenticating to a registry")
+	flags.StringVar(&pullOptions.OverrideArch, "override-arch", "", "Use `ARCH` instead of the architecture of the machine for choosing images")
+	flags.StringVar(&pullOptions.OverrideOS, "override-os", "", "Use `OS` instead of the running OS for choosing images")
+	flags.BoolVarP(&pullOptions.Quiet, "quiet", "q", false, "Suppress output information when pulling images")
+	flags.StringVar(&pullOptions.SignaturePolicy, "signature-policy", "", "`Pathname` of signature policy file (not usually used)")
+	flags.BoolVar(&pullOptions.TLSVerifyCLI, "tls-verify", true, "Require HTTPS and verify certificates when contacting registries")
+
+	if registry.IsRemote() {
+		_ = flags.MarkHidden("authfile")
+		_ = flags.MarkHidden("cert-dir")
+		_ = flags.MarkHidden("signature-policy")
+		_ = flags.MarkHidden("tls-verify")
+	}
+}
+
+// imagePull is implement the command for pulling images.
+func imagePull(cmd *cobra.Command, args []string) error {
+	// Sanity check input.
+	if len(args) == 0 {
+		return errors.Errorf("an image name must be specified")
+	}
+	if len(args) > 1 {
+		return errors.Errorf("too many arguments. Requires exactly 1")
+	}
+
+	// Start tracing if requested.
+	if cmd.Flags().Changed("trace") {
+		span, _ := opentracing.StartSpanFromContext(registry.GetContext(), "pullCmd")
+		defer span.Finish()
+	}
+
+	pullOptsAPI := pullOptions.ImagePullOptions
+	// TLS verification in c/image is controlled via a `types.OptionalBool`
+	// which allows for distinguishing among set-true, set-false, unspecified
+	// which is important to implement a sane way of dealing with defaults of
+	// boolean CLI flags.
+	if cmd.Flags().Changed("tls-verify") {
+		pullOptsAPI.TLSVerify = types.NewOptionalBool(pullOptions.TLSVerifyCLI)
+	}
+
+	// Let's do all the remaining Yoga in the API to prevent us from
+	// scattering logic across (too) many parts of the code.
+	pullReport, err := registry.ImageEngine().Pull(registry.GetContext(), args[0], pullOptsAPI)
+	if err != nil {
+		return err
+	}
+
+	if len(pullReport.Images) > 1 {
+		fmt.Println("Pulled Images:")
+	}
+	for _, img := range pullReport.Images {
+		fmt.Println(img)
+	}
+
+	return nil
+}

pkg/api/handlers/libpod/images.go

@@ -303,6 +303,10 @@ func ImagesImport(w http.ResponseWriter, r *http.Request) {
 	utils.WriteResponse(w, http.StatusOK, handlers.LibpodImagesImportReport{ID: importedImage})
 }
 
+// ImagesPull is the v2 libpod endpoint for pulling images.  Note that the
+// mandatory `reference` must be a reference to a registry (i.e., of docker
+// transport or be normalized to one).  Other transports are rejected as they
+// do not make sense in a remote context.
 func ImagesPull(w http.ResponseWriter, r *http.Request) {
 	runtime := r.Context().Value("runtime").(*libpod.Runtime)
 	decoder := r.Context().Value("decoder").(*schema.Decoder)
@@ -328,10 +332,11 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	// Enforce the docker transport.  This is just a precaution as some callers
-	// might accustomed to using the "transport:reference" notation.  Using
+	// might be accustomed to using the "transport:reference" notation.  Using
 	// another than the "docker://" transport does not really make sense for a
 	// remote case. For loading tarballs, the load and import endpoints should
 	// be used.
+	dockerPrefix := fmt.Sprintf("%s://", docker.Transport.Name())
 	imageRef, err := alltransports.ParseImageName(query.Reference)
 	if err == nil && imageRef.Transport().Name() != docker.Transport.Name() {
 		utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest,
@@ -339,7 +344,7 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) {
 		return
 	} else if err != nil {
 		origErr := err
-		imageRef, err = alltransports.ParseImageName(fmt.Sprintf("%s://%s", docker.Transport.Name(), query.Reference))
+		imageRef, err = alltransports.ParseImageName(fmt.Sprintf("%s%s", dockerPrefix, query.Reference))
 		if err != nil {
 			utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest,
 				errors.Wrapf(origErr, "reference %q must be a docker reference", query.Reference))
@@ -347,16 +352,19 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	// Trim the docker-transport prefix.
+	rawImage := strings.TrimPrefix(query.Reference, dockerPrefix)
+
 	// all-tags doesn't work with a tagged reference, so let's check early
-	namedRef, err := reference.Parse(query.Reference)
+	namedRef, err := reference.Parse(rawImage)
 	if err != nil {
 		utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest,
-			errors.Wrapf(err, "error parsing reference %q", query.Reference))
+			errors.Wrapf(err, "error parsing reference %q", rawImage))
 		return
 	}
 	if _, isTagged := namedRef.(reference.Tagged); isTagged && query.AllTags {
 		utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest,
-			errors.Errorf("reference %q must not have a tag for all-tags", query.Reference))
+			errors.Errorf("reference %q must not have a tag for all-tags", rawImage))
 		return
 	}
 

pkg/bindings/images/images.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 	"strconv"
 
+	"github.com/containers/image/v5/types"
 	"github.com/containers/libpod/pkg/api/handlers"
 	"github.com/containers/libpod/pkg/bindings"
 	"github.com/containers/libpod/pkg/domain/entities"
@@ -229,3 +230,41 @@ func Import(ctx context.Context, changes []string, message, reference, u *string
 	}
 	return id.ID, response.Process(&id)
 }
+
+// Pull is the binding for libpod's v2 endpoints for pulling images.  Note that
+// `rawImage` must be a reference to a registry (i.e., of docker transport or be
+// normalized to one).  Other transports are rejected as they do not make sense
+// in a remote context.
+func Pull(ctx context.Context, rawImage string, options entities.ImagePullOptions) ([]string, error) {
+	conn, err := bindings.GetClient(ctx)
+	if err != nil {
+		return nil, err
+	}
+	params := url.Values{}
+	params.Set("reference", rawImage)
+	params.Set("credentials", options.Credentials)
+	params.Set("overrideArch", options.OverrideArch)
+	params.Set("overrideOS", options.OverrideOS)
+	if options.TLSVerify != types.OptionalBoolUndefined {
+		val := bool(options.TLSVerify == types.OptionalBoolTrue)
+		params.Set("tlsVerify", strconv.FormatBool(val))
+	}
+	params.Set("allTags", strconv.FormatBool(options.AllTags))
+
+	response, err := conn.DoRequest(nil, http.MethodPost, "/images/pull", params)
+	if err != nil {
+		return nil, err
+	}
+
+	reports := []handlers.LibpodImagesPullReport{}
+	if err := response.Process(&reports); err != nil {
+		return nil, err
+	}
+
+	pulledImages := []string{}
+	for _, r := range reports {
+		pulledImages = append(pulledImages, r.ID)
+	}
+
+	return pulledImages, nil
+}

pkg/bindings/test/images_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/containers/libpod/pkg/bindings"
 	"github.com/containers/libpod/pkg/bindings/containers"
 	"github.com/containers/libpod/pkg/bindings/images"
+	"github.com/containers/libpod/pkg/domain/entities"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/onsi/gomega/gexec"
@@ -353,4 +354,24 @@ var _ = Describe("Podman images", func() {
 		Expect(results).To(ContainElement("docker.io/library/alpine:latest"))
 	})
 
+	// TODO: we really need to extent to pull tests once we have a more sophisticated CI.
+	It("Image Pull", func() {
+		rawImage := "docker.io/library/busybox:latest"
+
+		pulledImages, err := images.Pull(bt.conn, rawImage, entities.ImagePullOptions{})
+		Expect(err).To(BeNil())
+		Expect(len(pulledImages)).To(Equal(1))
+
+		exists, err := images.Exists(bt.conn, rawImage)
+		Expect(err).To(BeNil())
+		Expect(exists).To(BeTrue())
+
+		// Make sure the normalization AND the full-transport reference works.
+		_, err = images.Pull(bt.conn, "docker://"+rawImage, entities.ImagePullOptions{})
+		Expect(err).To(BeNil())
+
+		// The v2 endpoint only supports the docker transport.  Let's see if that's really true.
+		_, err = images.Pull(bt.conn, "bogus-transport:bogus.com/image:reference", entities.ImagePullOptions{})
+		Expect(err).To(Not(BeNil()))
+	})
 })

pkg/domain/entities/engine_image.go

@@ -10,4 +10,5 @@ type ImageEngine interface {
 	History(ctx context.Context, nameOrId string, opts ImageHistoryOptions) (*ImageHistoryReport, error)
 	List(ctx context.Context, opts ImageListOptions) ([]*ImageSummary, error)
 	Prune(ctx context.Context, opts ImagePruneOptions) (*ImagePruneReport, error)
+	Pull(ctx context.Context, rawImage string, opts ImagePullOptions) (*ImagePullReport, error)
 }

pkg/domain/entities/images.go

@@ -4,6 +4,7 @@ import (
 	"net/url"
 
 	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/types"
 	docker "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/opencontainers/go-digest"
@@ -117,6 +118,39 @@ type ImageInspectOptions struct {
 	Latest     bool   `json:",omitempty"`
 }
 
+// ImagePullOptions are the arguments for pulling images.
+type ImagePullOptions struct {
+	// AllTags can be specified to pull all tags of the spiecifed image. Note
+	// that this only works if the specified image does not include a tag.
+	AllTags bool
+	// Authfile is the path to the authentication file. Ignored for remote
+	// calls.
+	Authfile string
+	// CertDir is the path to certificate directories.  Ignored for remote
+	// calls.
+	CertDir string
+	// Credentials for authenticating against the registry in the format
+	// USERNAME:PASSWORD.
+	Credentials string
+	// OverrideArch will overwrite the local architecture for image pulls.
+	OverrideArch string
+	// OverrideOS will overwrite the local operating system (OS) for image
+	// pulls.
+	OverrideOS string
+	// Quiet can be specified to suppress pull progress when pulling.  Ignored
+	// for remote calls.
+	Quiet bool
+	// SignaturePolicy to use when pulling.  Ignored for remote calls.
+	SignaturePolicy string
+	// TLSVerify to enable/disable HTTPS and certificate verification.
+	TLSVerify types.OptionalBool
+}
+
+// ImagePullReport is the response from pulling one or more images.
+type ImagePullReport struct {
+	Images []string
+}
+
 type ImageListOptions struct {
 	All     bool       `json:"all" schema:"all"`
 	Filter  []string   `json:"Filter,omitempty"`

pkg/domain/infra/abi/images.go

@@ -5,11 +5,22 @@ package abi
 import (
 	"context"
 	"fmt"
+	"io"
+	"os"
+	"strings"
 
+	"github.com/containers/image/v5/docker"
+	dockerarchive "github.com/containers/image/v5/docker/archive"
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/transports/alltransports"
+	"github.com/containers/image/v5/types"
+	"github.com/containers/libpod/libpod/image"
 	libpodImage "github.com/containers/libpod/libpod/image"
 	"github.com/containers/libpod/pkg/domain/entities"
+	"github.com/containers/libpod/pkg/util"
 	"github.com/containers/storage"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 func (ir *ImageEngine) Exists(_ context.Context, nameOrId string) (*entities.BoolReport, error) {
@@ -134,6 +145,95 @@ func ToDomainHistoryLayer(layer *libpodImage.History) entities.ImageHistoryLayer
 	return l
 }
 
+func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, options entities.ImagePullOptions) (*entities.ImagePullReport, error) {
+	var writer io.Writer
+	if !options.Quiet {
+		writer = os.Stderr
+	}
+
+	dockerPrefix := fmt.Sprintf("%s://", docker.Transport.Name())
+	imageRef, err := alltransports.ParseImageName(rawImage)
+	if err != nil {
+		imageRef, err = alltransports.ParseImageName(fmt.Sprintf("%s%s", dockerPrefix, rawImage))
+		if err != nil {
+			return nil, errors.Errorf("invalid image reference %q", rawImage)
+		}
+	}
+
+	// Special-case for docker-archive which allows multiple tags.
+	if imageRef.Transport().Name() == dockerarchive.Transport.Name() {
+		newImage, err := ir.Libpod.ImageRuntime().LoadFromArchiveReference(ctx, imageRef, options.SignaturePolicy, writer)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error pulling image %q", rawImage)
+		}
+		return &entities.ImagePullReport{Images: []string{newImage[0].ID()}}, nil
+	}
+
+	var registryCreds *types.DockerAuthConfig
+	if options.Credentials != "" {
+		creds, err := util.ParseRegistryCreds(options.Credentials)
+		if err != nil {
+			return nil, err
+		}
+		registryCreds = creds
+	}
+	dockerRegistryOptions := image.DockerRegistryOptions{
+		DockerRegistryCreds:         registryCreds,
+		DockerCertPath:              options.CertDir,
+		OSChoice:                    options.OverrideOS,
+		ArchitectureChoice:          options.OverrideArch,
+		DockerInsecureSkipTLSVerify: options.TLSVerify,
+	}
+
+	if !options.AllTags {
+		newImage, err := ir.Libpod.ImageRuntime().New(ctx, rawImage, options.SignaturePolicy, options.Authfile, writer, &dockerRegistryOptions, image.SigningOptions{}, nil, util.PullImageAlways)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error pulling image %q", rawImage)
+		}
+		return &entities.ImagePullReport{Images: []string{newImage.ID()}}, nil
+	}
+
+	// --all-tags requires the docker transport
+	if imageRef.Transport().Name() != docker.Transport.Name() {
+		return nil, errors.New("--all-tags requires docker transport")
+	}
+
+	// Trim the docker-transport prefix.
+	rawImage = strings.TrimPrefix(rawImage, docker.Transport.Name())
+
+	// all-tags doesn't work with a tagged reference, so let's check early
+	namedRef, err := reference.Parse(rawImage)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error parsing %q", rawImage)
+	}
+	if _, isTagged := namedRef.(reference.Tagged); isTagged {
+		return nil, errors.New("--all-tags requires a reference without a tag")
+
+	}
+
+	systemContext := image.GetSystemContext("", options.Authfile, false)
+	tags, err := docker.GetRepositoryTags(ctx, systemContext, imageRef)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error getting repository tags")
+	}
+
+	var foundIDs []string
+	for _, tag := range tags {
+		name := rawImage + ":" + tag
+		newImage, err := ir.Libpod.ImageRuntime().New(ctx, name, options.SignaturePolicy, options.Authfile, writer, &dockerRegistryOptions, image.SigningOptions{}, nil, util.PullImageAlways)
+		if err != nil {
+			logrus.Errorf("error pulling image %q", name)
+			continue
+		}
+		foundIDs = append(foundIDs, newImage.ID())
+	}
+
+	if len(tags) != len(foundIDs) {
+		return nil, errors.Errorf("error pulling image %q", rawImage)
+	}
+	return &entities.ImagePullReport{Images: foundIDs}, nil
+}
+
 // func (r *imageRuntime) Delete(ctx context.Context, nameOrId string, opts entities.ImageDeleteOptions) (*entities.ImageDeleteReport, error) {
 // 	image, err := r.libpod.ImageEngine().NewFromLocal(nameOrId)
 // 	if err != nil {

pkg/domain/infra/tunnel/images.go

@@ -85,3 +85,11 @@ func (ir *ImageEngine) Prune(ctx context.Context, opts entities.ImagePruneOption
 	}
 	return &report, nil
 }
+
+func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, options entities.ImagePullOptions) (*entities.ImagePullReport, error) {
+	pulledImages, err := images.Pull(ir.ClientCxt, rawImage, options)
+	if err != nil {
+		return nil, err
+	}
+	return &entities.ImagePullReport{Images: pulledImages}, nil
+}