Merge pull request #2348 from baude/remotepullverbose

podman-remote pull

API.md

@@ -91,9 +91,9 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in
 
 [func PausePod(name: string) string](#PausePod)
 
-[func PullImage(name: string, certDir: string, creds: string, signaturePolicy: string, tlsVerify: ?bool) string](#PullImage)
+[func PullImage(name: string, certDir: string, creds: string, signaturePolicy: string, tlsVerify: ) MoreResponse](#PullImage)
 
-[func PushImage(name: string, tag: string, tlsverify: ?bool, signaturePolicy: string, creds: string, certDir: string, compress: bool, format: string, removeSignatures: bool, signBy: string) MoreResponse](#PushImage)
+[func PushImage(name: string, tag: string, tlsverify: , signaturePolicy: string, creds: string, certDir: string, compress: bool, format: string, removeSignatures: bool, signBy: string) MoreResponse](#PushImage)
 
 [func ReceiveFile(path: string, delete: bool) int](#ReceiveFile)
 
@@ -107,7 +107,7 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in
 
 [func RestartPod(name: string) string](#RestartPod)
 
-[func SearchImages(query: string, limit: int, tlsVerify: ?bool) ImageSearchResult](#SearchImages)
+[func SearchImages(query: string, limit: , tlsVerify: ) ImageSearchResult](#SearchImages)
 
 [func SendFile(type: string, length: int) string](#SendFile)
 
@@ -753,20 +753,14 @@ $ varlink call -m unix:/run/podman/io.podman/io.podman.PausePod '{"name": "fooba
 ### <a name="PullImage"></a>func PullImage
 <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
 
-method PullImage(name: [string](https://godoc.org/builtin#string), certDir: [string](https://godoc.org/builtin#string), creds: [string](https://godoc.org/builtin#string), signaturePolicy: [string](https://godoc.org/builtin#string), tlsVerify: [bool](https://godoc.org/builtin#bool)) [string](https://godoc.org/builtin#string)</div>
+method PullImage(name: [string](https://godoc.org/builtin#string), certDir: [string](https://godoc.org/builtin#string), creds: [string](https://godoc.org/builtin#string), signaturePolicy: [string](https://godoc.org/builtin#string), tlsVerify: [](#)) [MoreResponse](#MoreResponse)</div>
-PullImage pulls an image from a repository to local storage.  After the pull is successful, the ID of the image
+PullImage pulls an image from a repository to local storage.  After a successful pull, the image id and logs
-is returned.
+are returned as a [MoreResponse](#MoreResponse).  This connection also will handle a WantsMores request to send
-#### Example
+status as it occurs.
-~~~
-$ varlink call -m unix:/run/podman/io.podman/io.podman.PullImage '{"name": "registry.fedoraproject.org/fedora"}'
-{
-  "id": "426866d6fa419873f97e5cbd320eeb22778244c1dfffa01c944db3114f55772e"
-}
-~~~
 ### <a name="PushImage"></a>func PushImage
 <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
 
-method PushImage(name: [string](https://godoc.org/builtin#string), tag: [string](https://godoc.org/builtin#string), tlsverify: [bool](https://godoc.org/builtin#bool), signaturePolicy: [string](https://godoc.org/builtin#string), creds: [string](https://godoc.org/builtin#string), certDir: [string](https://godoc.org/builtin#string), compress: [bool](https://godoc.org/builtin#bool), format: [string](https://godoc.org/builtin#string), removeSignatures: [bool](https://godoc.org/builtin#bool), signBy: [string](https://godoc.org/builtin#string)) [MoreResponse](#MoreResponse)</div>
+method PushImage(name: [string](https://godoc.org/builtin#string), tag: [string](https://godoc.org/builtin#string), tlsverify: [](#), signaturePolicy: [string](https://godoc.org/builtin#string), creds: [string](https://godoc.org/builtin#string), certDir: [string](https://godoc.org/builtin#string), compress: [bool](https://godoc.org/builtin#bool), format: [string](https://godoc.org/builtin#string), removeSignatures: [bool](https://godoc.org/builtin#bool), signBy: [string](https://godoc.org/builtin#string)) [MoreResponse](#MoreResponse)</div>
 PushImage takes three input arguments: the name or ID of an image, the fully-qualified destination name of the image,
 and a boolean as to whether tls-verify should be used (with false disabling TLS, not affecting the default behavior).
 It will return an [ImageNotFound](#ImageNotFound) error if
@@ -851,7 +845,7 @@ $ varlink call -m unix:/run/podman/io.podman/io.podman.RestartPod '{"name": "135
 ### <a name="SearchImages"></a>func SearchImages
 <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
 
-method SearchImages(query: [string](https://godoc.org/builtin#string), limit: [](#)) [ImageSearchResult](#ImageSearchResult)</div>
+method SearchImages(query: [string](https://godoc.org/builtin#string), limit: [](#), tlsVerify: [](#)) [ImageSearchResult](#ImageSearchResult)</div>
 SearchImages searches available registries for images that contain the
 contents of "query" in their name. If "limit" is given, limits the amount of
 search results per registry.
@@ -1602,7 +1596,7 @@ pull [bool](https://godoc.org/builtin#bool)
 
 signaturePolicyPath [string](https://godoc.org/builtin#string)
 
-tlsVerify [bool](https://godoc.org/builtin#bool)
+tlsVerify [](#)
 
 label [string](https://godoc.org/builtin#string)
 

cmd/podman/varlink/io.podman.varlink

@@ -719,16 +719,10 @@ method ImportImage(source: string, reference: string, message: string, changes:
 # error will be returned. See also [ImportImage](ImportImage).
 method ExportImage(name: string, destination: string, compress: bool, tags: []string) -> (image: string)
 
-# PullImage pulls an image from a repository to local storage.  After the pull is successful, the ID of the image
+# PullImage pulls an image from a repository to local storage.  After a successful pull, the image id and logs
-# is returned.
+# are returned as a [MoreResponse](#MoreResponse).  This connection also will handle a WantsMores request to send
-# #### Example
+# status as it occurs.
-# ~~~
+method PullImage(name: string, certDir: string, creds: string, signaturePolicy: string, tlsVerify: ?bool) -> (reply: MoreResponse)
-# $ varlink call -m unix:/run/podman/io.podman/io.podman.PullImage '{"name": "registry.fedoraproject.org/fedora"}'
-# {
-#   "id": "426866d6fa419873f97e5cbd320eeb22778244c1dfffa01c944db3114f55772e"
-# }
-# ~~~
-method PullImage(name: string, certDir: string, creds: string, signaturePolicy: string, tlsVerify: ?bool) -> (id: string)
 
 # CreatePod creates a new empty pod.  It uses a [PodCreate](#PodCreate) type for input.
 # On success, the ID of the newly created pod will be returned.

libpod/adapter/runtime_remote.go

@@ -161,14 +161,30 @@ func (r *LocalRuntime) NewImageFromLocal(name string) (*ContainerImage, error) {
 
 // LoadFromArchiveReference creates an image from a local archive
 func (r *LocalRuntime) LoadFromArchiveReference(ctx context.Context, srcRef types.ImageReference, signaturePolicyPath string, writer io.Writer) ([]*ContainerImage, error) {
+	var iid string
 	// TODO We need to find a way to leak certDir, creds, and the tlsverify into this function, normally this would
 	// come from cli options but we don't want want those in here either.
 	tlsverify := true
-	imageID, err := iopodman.PullImage().Call(r.Conn, srcRef.DockerReference().String(), "", "", signaturePolicyPath, &tlsverify)
+	reply, err := iopodman.PullImage().Send(r.Conn, varlink.More, srcRef.DockerReference().String(), "", "", signaturePolicyPath, &tlsverify)
 	if err != nil {
 		return nil, err
 	}
-	newImage, err := r.NewImageFromLocal(imageID)
+
+	for {
+		responses, flags, err := reply()
+		if err != nil {
+			return nil, err
+		}
+		for _, line := range responses.Logs {
+			fmt.Print(line)
+		}
+		iid = responses.Id
+		if flags&varlink.Continues == 0 {
+			break
+		}
+	}
+
+	newImage, err := r.NewImageFromLocal(iid)
 	if err != nil {
 		return nil, err
 	}
@@ -177,6 +193,7 @@ func (r *LocalRuntime) LoadFromArchiveReference(ctx context.Context, srcRef type
 
 // New calls into local storage to look for an image in local storage or to pull it
 func (r *LocalRuntime) New(ctx context.Context, name, signaturePolicyPath, authfile string, writer io.Writer, dockeroptions *image.DockerRegistryOptions, signingoptions image.SigningOptions, forcePull bool, label *string) (*ContainerImage, error) {
+	var iid string
 	if label != nil {
 		return nil, errors.New("the remote client function does not support checking a remote image for a label")
 	}
@@ -194,11 +211,24 @@ func (r *LocalRuntime) New(ctx context.Context, name, signaturePolicyPath, authf
 		tlsVerifyPtr = &tlsVerify
 	}
 
-	imageID, err := iopodman.PullImage().Call(r.Conn, name, dockeroptions.DockerCertPath, "", signaturePolicyPath, tlsVerifyPtr)
+	reply, err := iopodman.PullImage().Send(r.Conn, varlink.More, name, dockeroptions.DockerCertPath, "", signaturePolicyPath, tlsVerifyPtr)
 	if err != nil {
 		return nil, err
 	}
-	newImage, err := r.NewImageFromLocal(imageID)
+	for {
+		responses, flags, err := reply()
+		if err != nil {
+			return nil, err
+		}
+		for _, line := range responses.Logs {
+			fmt.Print(line)
+		}
+		iid = responses.Id
+		if flags&varlink.Continues == 0 {
+			break
+		}
+	}
+	newImage, err := r.NewImageFromLocal(iid)
 	if err != nil {
 		return nil, err
 	}

pkg/varlinkapi/images.go

@@ -137,7 +137,7 @@ func (i *LibpodAPI) BuildImage(call iopodman.VarlinkCall, config iopodman.BuildI
 	logrus.Debugf("untar of %s successful", contextDir)
 
 	// All output (stdout, stderr) is captured in output as well
-	output := bytes.NewBuffer([]byte{})
+	var output bytes.Buffer
 
 	commonOpts := &buildah.CommonBuildOptions{
 		AddHost:      config.BuildOptions.AddHosts,
@@ -170,20 +170,20 @@ func (i *LibpodAPI) BuildImage(call iopodman.VarlinkCall, config iopodman.BuildI
 		Compression:             stringCompressionToArchiveType(config.Compression),
 		ContextDirectory:        newContextDir,
 		DefaultMountsFilePath:   config.DefaultsMountFilePath,
-		Err:                     output,
+		Err:                     &output,
 		ForceRmIntermediateCtrs: config.ForceRmIntermediateCtrs,
 		IIDFile:                 config.Iidfile,
 		Labels:                  config.Label,
 		Layers:                  config.Layers,
 		NoCache:                 config.Nocache,
-		Out:                     output,
+		Out:                     &output,
 		Output:                  config.Output,
 		NamespaceOptions:        namespace,
 		OutputFormat:            config.OutputFormat,
 		PullPolicy:              stringPullPolicyToType(config.PullPolicy),
 		Quiet:                   config.Quiet,
 		RemoveIntermediateCtrs:  config.RemoteIntermediateCtrs,
-		ReportWriter:            output,
+		ReportWriter:            &output,
 		RuntimeArgs:             config.RuntimeArgs,
 		SignaturePolicyPath:     config.SignaturePolicyPath,
 		Squash:                  config.Squash,
@@ -208,7 +208,13 @@ func (i *LibpodAPI) BuildImage(call iopodman.VarlinkCall, config iopodman.BuildI
 		newPathDockerFiles = append(newPathDockerFiles, filepath.Join(newContextDir, base))
 	}
 
-	c := build(i.Runtime, options, newPathDockerFiles)
+	c := make(chan error)
+	go func() {
+		err := i.Runtime.Build(getContext(), options, newPathDockerFiles...)
+		c <- err
+		close(c)
+	}()
+
 	var log []string
 	done := false
 	for {
@@ -257,17 +263,6 @@ func (i *LibpodAPI) BuildImage(call iopodman.VarlinkCall, config iopodman.BuildI
 	return call.ReplyBuildImage(br)
 }
 
-func build(runtime *libpod.Runtime, options imagebuildah.BuildOptions, dockerfiles []string) chan error {
-	c := make(chan error)
-	go func() {
-		err := runtime.Build(getContext(), options, dockerfiles...)
-		c <- err
-		close(c)
-	}()
-
-	return c
-}
-
 // InspectImage returns an image's inspect information as a string that can be serialized.
 // Requires an image ID or name
 func (i *LibpodAPI) InspectImage(call iopodman.VarlinkCall, name string) error {
@@ -622,24 +617,74 @@ func (i *LibpodAPI) PullImage(call iopodman.VarlinkCall, name string, certDir, c
 
 	so := image.SigningOptions{}
 
-	if strings.HasPrefix(name, dockerarchive.Transport.Name()+":") {
+	if call.WantsMore() {
-		srcRef, err := alltransports.ParseImageName(name)
+		call.Continues = true
-		if err != nil {
-			return errors.Wrapf(err, "error parsing %q", name)
-		}
-		newImage, err := i.Runtime.ImageRuntime().LoadFromArchiveReference(getContext(), srcRef, signaturePolicy, nil)
-		if err != nil {
-			return errors.Wrapf(err, "error pulling image from %q", name)
-		}
-		imageID = newImage[0].ID()
-	} else {
-		newImage, err := i.Runtime.ImageRuntime().New(getContext(), name, signaturePolicy, "", nil, &dockerRegistryOptions, so, false, nil)
-		if err != nil {
-			return call.ReplyErrorOccurred(fmt.Sprintf("unable to pull %s: %s", name, err.Error()))
-		}
-		imageID = newImage.ID()
 	}
-	return call.ReplyPullImage(imageID)
+	output := bytes.NewBuffer([]byte{})
+	c := make(chan error)
+	go func() {
+		//err := newImage.PushImageToHeuristicDestination(getContext(), destname, manifestType, "", signaturePolicy, output, compress, so, &dockerRegistryOptions, nil)
+		if strings.HasPrefix(name, dockerarchive.Transport.Name()+":") {
+			srcRef, err := alltransports.ParseImageName(name)
+			if err != nil {
+				c <- errors.Wrapf(err, "error parsing %q", name)
+			}
+			newImage, err := i.Runtime.ImageRuntime().LoadFromArchiveReference(getContext(), srcRef, signaturePolicy, output)
+			if err != nil {
+				c <- errors.Wrapf(err, "error pulling image from %q", name)
+			}
+			imageID = newImage[0].ID()
+		} else {
+			newImage, err := i.Runtime.ImageRuntime().New(getContext(), name, signaturePolicy, "", output, &dockerRegistryOptions, so, false, nil)
+			if err != nil {
+				c <- errors.Wrapf(err, "unable to pull %s", name)
+			}
+			imageID = newImage.ID()
+		}
+		c <- nil
+		close(c)
+	}()
+
+	var log []string
+	done := false
+	for {
+		line, err := output.ReadString('\n')
+		if err == nil {
+			log = append(log, line)
+			continue
+		} else if err == io.EOF {
+			select {
+			case err := <-c:
+				if err != nil {
+					logrus.Errorf("reading of output during pull failed for %s", name)
+					return call.ReplyErrorOccurred(err.Error())
+				}
+				done = true
+			default:
+				if !call.WantsMore() {
+					time.Sleep(1 * time.Second)
+					break
+				}
+				br := iopodman.MoreResponse{
+					Logs: log,
+				}
+				call.ReplyPullImage(br)
+				log = []string{}
+			}
+		} else {
+			return call.ReplyErrorOccurred(err.Error())
+		}
+		if done {
+			break
+		}
+	}
+	call.Continues = false
+
+	br := iopodman.MoreResponse{
+		Logs: log,
+		Id:   imageID,
+	}
+	return call.ReplyPullImage(br)
 }
 
 // ImageExists returns bool as to whether the input image exists in local storage