Show c/storage (Buildah/CRI-O) containers in ps

The `podman ps --all` command will now show containers that
are under the control of other c/storage container systems and
the new `ps --storage` option will show only containers that are
in c/storage but are not controlled by libpod.

In the below examples, the '*working-container' entries were created
by Buildah.

```
podman ps -a
CONTAINER ID  IMAGE                             COMMAND  CREATED       STATUS                   PORTS  NAMES
9257ef8c786c  docker.io/library/busybox:latest  ls /etc  8 hours ago   Exited (0) 8 hours ago          gifted_jang
d302c81856da  docker.io/library/busybox:latest  buildah  30 hours ago  storage                         busybox-working-container
7a5a7b099d33  localhost/tom:latest              ls -alF  30 hours ago  Exited (0) 30 hours ago         hopeful_hellman
01d601fca090  localhost/tom:latest              ls -alf  30 hours ago  Exited (1) 30 hours ago         determined_panini
ee58f429ff26  localhost/tom:latest              buildah  33 hours ago  storage                         alpine-working-container

podman ps --external
CONTAINER ID  IMAGE                             COMMAND  CREATED       STATUS    PORTS  NAMES
d302c81856da  docker.io/library/busybox:latest  buildah  30 hours ago  external         busybox-working-container
ee58f429ff26  localhost/tom:latest              buildah  33 hours ago  external         alpine-working-container

```
Signed-off-by: TomSweeneyRedHat <tsweeney@redhat.com>
Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

cmd/podman/containers/ps.go

@@ -13,6 +13,7 @@ import (
 	tm "github.com/buger/goterm"
 	"github.com/containers/buildah/pkg/formats"
 	"github.com/containers/podman/v2/cmd/podman/registry"
+	"github.com/containers/podman/v2/cmd/podman/utils"
 	"github.com/containers/podman/v2/cmd/podman/validate"
 	"github.com/containers/podman/v2/pkg/domain/entities"
 	"github.com/cri-o/ocicni/pkg/ocicni"
@@ -56,9 +57,9 @@ func init() {
 func listFlagSet(flags *pflag.FlagSet) {
 	flags.BoolVarP(&listOpts.All, "all", "a", false, "Show all the containers, default is only running containers")
 	flags.StringSliceVarP(&filters, "filter", "f", []string{}, "Filter output based on conditions given")
+	flags.BoolVar(&listOpts.Storage, "storage", false, "Show containers in storage not controlled by Podman")
 	flags.StringVar(&listOpts.Format, "format", "", "Pretty-print containers to JSON or using a Go template")
 	flags.IntVarP(&listOpts.Last, "last", "n", -1, "Print the n last created containers (all states)")
-	flags.BoolVar(&listOpts.Namespace, "namespace", false, "Display namespace information")
 	flags.BoolVar(&listOpts.Namespace, "ns", false, "Display namespace information")
 	flags.BoolVar(&noTrunc, "no-trunc", false, "Display the extended information")
 	flags.BoolVarP(&listOpts.Pod, "pod", "p", false, "Print the ID and name of the pod the containers are associated with")
@@ -69,6 +70,7 @@ func listFlagSet(flags *pflag.FlagSet) {
 
 	sort := validate.Value(&listOpts.Sort, "command", "created", "id", "image", "names", "runningfor", "size", "status")
 	flags.Var(sort, "sort", "Sort output by: "+sort.Choices())
+	flags.SetNormalizeFunc(utils.AliasFlags)
 }
 func checkFlags(c *cobra.Command) error {
 	// latest, and last are mutually exclusive.
@@ -102,6 +104,14 @@ func checkFlags(c *cobra.Command) error {
 	if listOpts.Watch > 0 && listOpts.Latest {
 		return errors.New("the watch and latest flags cannot be used together")
 	}
+	cfg := registry.PodmanConfig()
+	if cfg.Engine.Namespace != "" {
+		if c.Flag("storage").Changed && listOpts.Storage {
+			return errors.New("--namespace and --storage flags can not both be set")
+		}
+		listOpts.Storage = false
+	}
+
 	return nil
 }
 

cmd/podman/utils/alias.go

@@ -19,6 +19,8 @@ func AliasFlags(f *pflag.FlagSet, name string) pflag.NormalizedName {
 		name = "network"
 	case "timeout":
 		name = "time"
+	case "namespace":
+		name = "ns"
 	}
 	return pflag.NormalizedName(name)
 }

completions/bash/podman

@@ -2679,6 +2679,7 @@ _podman_ps() {
 	  --pod -p
 	  --quiet -q
 	  --size -s
+	  --storage
 	  --namespace --ns
 	  --sync
      "

docs/source/markdown/podman-build.1.md

@@ -23,6 +23,8 @@ When the URL is an Containerfile, the Containerfile is downloaded to a temporary
 
 When a Git repository is set as the URL, the repository is cloned locally and then set as the context.
 
+NOTE: `podman build` uses code sourced from the `buildah` project to build container images.  This `buildah` code creates `buildah` containers for the `RUN` options in container storage. In certain situations, when the `podman build` crashes or users kill the `podman build` process, these external containers can be left in container storage. Use the `podman ps --all --storage` command to see these contaienrs. External containers can be removed with the `podman rm --storage` command.
+
 ## OPTIONS
 
 **--add-host**=*host*
@@ -804,7 +806,7 @@ If you are using a useradd command within a Containerfile with a large UID/GID,
 If you are using `useradd` within your build script, you should pass the `--no-log-init or -l` option to the `useradd` command.  This option tells useradd to stop creating the lastlog file.
 
 ## SEE ALSO
-podman(1), buildah(1), containers-registries.conf(5), crun(8), runc(8), useradd(8)
+podman(1), buildah(1), containers-registries.conf(5), crun(8), runc(8), useradd(8), podman-ps(1), podman-rm(1)
 
 ## HISTORY
 Aug 2020, Additional options and .dockerignore added by Dan Walsh <dwalsh@redhat.com>

docs/source/markdown/podman-ps.1.md

@@ -32,12 +32,18 @@ all the containers information.  By default it lists:
 
 **--all**, **-a**
 
-Show all the containers, default is only running containers
+Show all the containers created by Podman, default is only running containers.
+
+Note: Podman shares containers storage with other tools such as Buildah and CRI-O. In some cases these `external` containers might also exist in the same storage. Use the `--storage` option to see these external containers. External containers show the 'storage' status.
 
 **--pod**, **-p**
 
 Display the pods the containers are associated with
 
+**--storage**
+
+Display external containers that are not controlled by Podman but are stored in containers storage.  These external containers are generally created via other container technology such as Buildah or CRI-O and may depend on the same container images that Podman is also using.  External containers are denoted with either a 'buildah' or 'storage' in the COMMAND and STATUS column of the ps output. Only used with the --all option.
+
 **--no-trunc**
 
 Display the extended information
@@ -174,11 +180,20 @@ CONTAINER ID  IMAGE                            COMMAND    CREATED        STATUS
 
 ```
 
+```
+$ podman ps --storage -a
+CONTAINER ID  IMAGE                             COMMAND  CREATED      STATUS  PORTS  NAMES
+69ed779d8ef9f  redis:alpine  "redis-server"  25 hours ago  Created                   6379/tcp  k8s_container1_podsandbox1_redhat.test.crio_redhat-test-crio_1
+38a8a78596f9  docker.io/library/busybox:latest  buildah  2 hours ago  storage        busybox-working-container
+fd7b786b5c32  docker.io/library/alpine:latest   buildah  2 hours ago  storage        alpine-working-container
+f78620804e00  scratch                           buildah  2 hours ago  storage        working-container
+```
+
 ## ps
 Print a list of containers
 
 ## SEE ALSO
-podman(1)
+podman(1), buildah(1), crio(8)
 
 ## HISTORY
 August 2017, Originally compiled by Urvashi Mohnani <umohnani@redhat.com>

docs/source/markdown/podman-rm.1.md

@@ -45,9 +45,9 @@ The latest option is not supported on the remote client.
 
 **--storage**
 
-Remove the container from the storage library only.
+Remove external containers from the storage library.
-This is only possible with containers that are not present in libpod (cannot be seen by **podman ps**).
+This is only possible with containers that are not present in libpod can be seen by **podman ps --all --storage**).
-It is used to remove containers from **podman build** and **buildah**, and orphan containers which were only partially removed by **podman rm**.
+It is used to remove external containers from **podman build** and **buildah**, and orphan containers which were only partially removed by **podman rm**.
 The storage option conflicts with the **--all**, **--latest**, and **--volumes** options.
 
 **--volumes**, **-v**
@@ -96,7 +96,7 @@ $ podman rm -f --latest
   **125** The command fails for a reason other than container did not exist or is paused/running
 
 ## SEE ALSO
-podman(1), podman-image-rm(1)
+podman(1), podman-image-rm(1), podman-ps(1), podman-build(1)
 
 ## HISTORY
 August 2017, Originally compiled by Ryan Cole <rycole@redhat.com>

libpod/define/errors.go

@@ -161,4 +161,8 @@ var (
 	// ErrNetworkOnPodContainer indicates the user wishes to alter network attributes on a container
 	// in a pod.  This cannot be done as the infra container has all the network information
 	ErrNetworkOnPodContainer = errors.New("network cannot be configured when it is shared with a pod")
+
+	// ErrStoreNotInitialized indicates that the container storage was never
+	// initilized.
+	ErrStoreNotInitialized = errors.New("the container storage was never initilized")
 )

libpod/image/image.go

@@ -466,6 +466,14 @@ func (ir *Runtime) getImage(image string) (*storage.Image, error) {
 	return img, nil
 }
 
+func (ir *Runtime) ImageNames(id string) ([]string, error) {
+	myImage, err := ir.getImage(id)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error getting image %s ", id)
+	}
+	return myImage.Names, nil
+}
+
 // GetImages retrieves all images present in storage
 func (ir *Runtime) GetImages() ([]*Image, error) {
 	return ir.getImages(false)

libpod/image/prune.go

@@ -137,7 +137,7 @@ func (ir *Runtime) PruneImages(ctx context.Context, all bool, filter []string) (
 		}
 		if err := p.Remove(ctx, true); err != nil {
 			if errors.Cause(err) == storage.ErrImageUsedByContainer {
-				logrus.Warnf("Failed to prune image %s as it is in use: %v", p.ID(), err)
+				logrus.Warnf("Failed to prune image %s as it is in use: %v.\nA container associated with containers/storage i.e. Buildah, CRI-O, etc., maybe associated with this image.\nUsing the rmi command with the --force option will remove the container and image, but may cause failures for other dependent systems.", p.ID(), err)
 				continue
 			}
 			return nil, errors.Wrap(err, "failed to prune image")

libpod/runtime_ctr.go

@@ -8,11 +8,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containers/buildah"
 	"github.com/containers/common/pkg/config"
 	"github.com/containers/podman/v2/libpod/define"
 	"github.com/containers/podman/v2/libpod/events"
 	"github.com/containers/podman/v2/pkg/cgroups"
 	"github.com/containers/podman/v2/pkg/rootless"
+	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/stringid"
 	"github.com/docker/go-units"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
@@ -905,3 +907,34 @@ func (r *Runtime) PruneContainers(filterFuncs []ContainerFilter) (map[string]int
 	}
 	return prunedContainers, pruneErrors, nil
 }
+
+// StorageContainers returns a list of containers from containers/storage that
+// are not currently known to Podman.
+func (r *Runtime) StorageContainers() ([]storage.Container, error) {
+
+	if r.store == nil {
+		return nil, define.ErrStoreNotInitialized
+	}
+
+	storeContainers, err := r.store.Containers()
+	if err != nil {
+		return nil, errors.Wrapf(err, "error reading list of all storage containers")
+	}
+	retCtrs := []storage.Container{}
+	for _, container := range storeContainers {
+		exists, err := r.state.HasContainer(container.ID)
+		if err != nil && err != define.ErrNoSuchCtr {
+			return nil, errors.Wrapf(err, "failed to check if %s container exists in database", container.ID)
+		}
+		if exists {
+			continue
+		}
+		retCtrs = append(retCtrs, container)
+	}
+
+	return retCtrs, nil
+}
+
+func (r *Runtime) IsBuildahContainer(id string) (bool, error) {
+	return buildah.IsContainer(id, r.store)
+}

pkg/domain/entities/containers.go

@@ -282,6 +282,7 @@ type ContainerListOptions struct {
 	Quiet     bool
 	Size      bool
 	Sort      string
+	Storage   bool
 	Sync      bool
 	Watch     uint
 }

pkg/ps/ps.go

@@ -14,6 +14,7 @@ import (
 	lpfilters "github.com/containers/podman/v2/libpod/filters"
 	"github.com/containers/podman/v2/pkg/domain/entities"
 	psdefine "github.com/containers/podman/v2/pkg/ps/define"
+	"github.com/containers/storage"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -54,12 +55,12 @@ func GetContainerLists(runtime *libpod.Runtime, options entities.ContainerListOp
 		return nil, err
 	}
 	if options.Last > 0 {
-		// Sort the containers we got
+		// Sort the libpod containers
 		sort.Sort(SortCreateTime{SortContainers: cons})
 		// we should perform the lopping before we start getting
 		// the expensive information on containers
 		if options.Last < len(cons) {
-			cons = cons[len(cons)-options.Last:]
+			cons = cons[:options.Last]
 		}
 	}
 	for _, con := range cons {
@@ -68,7 +69,31 @@ func GetContainerLists(runtime *libpod.Runtime, options entities.ContainerListOp
 			return nil, err
 		}
 		pss = append(pss, listCon)
+	}
 
+	if options.All && options.Storage {
+		externCons, err := runtime.StorageContainers()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, con := range externCons {
+			listCon, err := ListStorageContainer(runtime, con, options)
+			if err != nil {
+				return nil, err
+			}
+			pss = append(pss, listCon)
+		}
+	}
+
+	// Sort the containers we got
+	sort.Sort(SortPSCreateTime{SortPSContainers: pss})
+
+	if options.Last > 0 {
+		// only return the "last" containers caller requested
+		if options.Last < len(pss) {
+			pss = pss[:options.Last]
+		}
 	}
 	return pss, nil
 }
@@ -199,6 +224,48 @@ func ListContainerBatch(rt *libpod.Runtime, ctr *libpod.Container, opts entities
 	return ps, nil
 }
 
+func ListStorageContainer(rt *libpod.Runtime, ctr storage.Container, opts entities.ContainerListOptions) (entities.ListContainer, error) {
+	name := "unknown"
+	if len(ctr.Names) > 0 {
+		name = ctr.Names[0]
+	}
+
+	ps := entities.ListContainer{
+		ID:      ctr.ID,
+		Created: ctr.Created.Unix(),
+		ImageID: ctr.ImageID,
+		State:   "storage",
+		Names:   []string{name},
+	}
+
+	buildahCtr, err := rt.IsBuildahContainer(ctr.ID)
+	if err != nil {
+		return ps, errors.Wrapf(err, "error determining buildah container for container %s", ctr.ID)
+	}
+
+	if buildahCtr {
+		ps.Command = []string{"buildah"}
+	} else {
+		ps.Command = []string{"storage"}
+	}
+
+	imageName := ""
+	if ctr.ImageID != "" {
+		names, err := rt.ImageRuntime().ImageNames(ctr.ImageID)
+		if err != nil {
+			return ps, err
+		}
+		if len(names) > 0 {
+			imageName = names[0]
+		}
+	} else if buildahCtr {
+		imageName = "scratch"
+	}
+
+	ps.Image = imageName
+	return ps, nil
+}
+
 func getNamespaceInfo(path string) (string, error) {
 	val, err := os.Readlink(path)
 	if err != nil {
@@ -223,5 +290,17 @@ func (a SortContainers) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
 type SortCreateTime struct{ SortContainers }
 
 func (a SortCreateTime) Less(i, j int) bool {
-	return a.SortContainers[i].CreatedTime().Before(a.SortContainers[j].CreatedTime())
+	return a.SortContainers[i].CreatedTime().After(a.SortContainers[j].CreatedTime())
+}
+
+// SortPSContainers helps us set-up ability to sort by createTime
+type SortPSContainers []entities.ListContainer
+
+func (a SortPSContainers) Len() int      { return len(a) }
+func (a SortPSContainers) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+type SortPSCreateTime struct{ SortPSContainers }
+
+func (a SortPSCreateTime) Less(i, j int) bool {
+	return a.SortPSContainers[i].Created > a.SortPSContainers[j].Created
 }