podman pod stats

add the ability to monitor container statistics in a pod.

Signed-off-by: baude <bbaude@redhat.com>

Closes: #1265
Approved by: rhatdan

cmd/podman/pod.go

@@ -18,6 +18,7 @@ Pods are a group of one or more containers sharing the same network, pid and ipc
 		podRestartCommand,
 		podRmCommand,
 		podStartCommand,
+		podStatsCommand,
 		podStopCommand,
 		podUnpauseCommand,
 	}

cmd/podman/pod_stats.go

@@ -0,0 +1,238 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"encoding/json"
+	tm "github.com/buger/goterm"
+	"github.com/containers/libpod/cmd/podman/formats"
+	"github.com/containers/libpod/cmd/podman/libpodruntime"
+	"github.com/containers/libpod/libpod"
+	"github.com/pkg/errors"
+	"github.com/ulule/deepcopier"
+	"github.com/urfave/cli"
+)
+
+var (
+	podStatsFlags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "all, a",
+			Usage: "show stats for all pods.  Only running pods are shown by default.",
+		},
+		cli.BoolFlag{
+			Name:  "no-stream",
+			Usage: "disable streaming stats and only pull the first result, default setting is false",
+		},
+		cli.BoolFlag{
+			Name:  "no-reset",
+			Usage: "disable resetting the screen between intervals",
+		},
+		cli.StringFlag{
+			Name:  "format",
+			Usage: "pretty-print container statistics to JSON or using a Go template",
+		}, LatestPodFlag,
+	}
+	podStatsDescription = "display a live stream of resource usage statistics for the containers in or more pods"
+	podStatsCommand     = cli.Command{
+		Name:                   "stats",
+		Usage:                  "Display percentage of CPU, memory, network I/O, block I/O and PIDs for containers in one or more pods",
+		Description:            podStatsDescription,
+		Flags:                  podStatsFlags,
+		Action:                 podStatsCmd,
+		ArgsUsage:              "[POD_NAME_OR_ID]",
+		UseShortOptionHandling: true,
+	}
+)
+
+func podStatsCmd(c *cli.Context) error {
+	var (
+		podFunc func() ([]*libpod.Pod, error)
+	)
+
+	format := c.String("format")
+	all := c.Bool("all")
+	latest := c.Bool("latest")
+	ctr := 0
+	if all {
+		ctr += 1
+	}
+	if latest {
+		ctr += 1
+	}
+	if len(c.Args()) > 0 {
+		ctr += 1
+	}
+
+	if ctr > 1 {
+		return errors.Errorf("--all, --latest and containers cannot be used together")
+	} else if ctr == 0 {
+		// If user didn't specify, imply --all
+		all = true
+	}
+
+	runtime, err := libpodruntime.GetRuntime(c)
+	if err != nil {
+		return errors.Wrapf(err, "could not get runtime")
+	}
+	defer runtime.Shutdown(false)
+
+	times := -1
+	if c.Bool("no-stream") {
+		times = 1
+	}
+
+	if len(c.Args()) > 0 {
+		podFunc = func() ([]*libpod.Pod, error) { return getPodsByList(c.Args(), runtime) }
+	} else if latest {
+		podFunc = func() ([]*libpod.Pod, error) {
+			latestPod, err := runtime.GetLatestPod()
+			if err != nil {
+				return nil, err
+			}
+			return []*libpod.Pod{latestPod}, err
+		}
+	} else if all {
+		podFunc = runtime.GetAllPods
+	} else {
+		podFunc = runtime.GetRunningPods
+	}
+
+	pods, err := podFunc()
+	if err != nil {
+		return errors.Wrapf(err, "unable to get a list of pods")
+	}
+
+	// First we need to get an initial pass of pod/ctr stats (these are not printed)
+	var podStats []*libpod.PodContainerStats
+	for _, p := range pods {
+		cons, err := p.AllContainersByID()
+		if err != nil {
+			return err
+		}
+		emptyStats := make(map[string]*libpod.ContainerStats)
+		// Iterate the pods container ids and make blank stats for them
+		for _, c := range cons {
+			emptyStats[c] = &libpod.ContainerStats{}
+		}
+		ps := libpod.PodContainerStats{
+			Pod:            p,
+			ContainerStats: emptyStats,
+		}
+		podStats = append(podStats, &ps)
+	}
+
+	// Create empty container stat results for our first pass
+	var previousPodStats []*libpod.PodContainerStats
+	for _, p := range pods {
+		cs := make(map[string]*libpod.ContainerStats)
+		pcs := libpod.PodContainerStats{
+			Pod:            p,
+			ContainerStats: cs,
+		}
+		previousPodStats = append(previousPodStats, &pcs)
+	}
+
+	step := 1
+	if times == -1 {
+		times = 1
+		step = 0
+	}
+
+	for i := 0; i < times; i += step {
+		var newStats []*libpod.PodContainerStats
+		for _, p := range pods {
+			prevStat := getPreviousPodContainerStats(p.ID(), previousPodStats)
+			newPodStats, err := p.GetPodStats(prevStat)
+			if errors.Cause(err) == libpod.ErrNoSuchPod {
+				continue
+			}
+			if err != nil {
+				return err
+			}
+			newPod := libpod.PodContainerStats{
+				Pod:            p,
+				ContainerStats: newPodStats,
+			}
+			newStats = append(newStats, &newPod)
+		}
+		//Output
+		if strings.ToLower(format) != formats.JSONString && !c.Bool("no-reset") {
+			tm.Clear()
+			tm.MoveCursor(1, 1)
+			tm.Flush()
+		}
+		if strings.ToLower(format) == formats.JSONString {
+			outputJson(newStats)
+
+		} else {
+			outputToStdOut(newStats)
+		}
+		time.Sleep(time.Second)
+		previousPodStats := new([]*libpod.PodContainerStats)
+		deepcopier.Copy(newStats).To(previousPodStats)
+		pods, err = podFunc()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func outputToStdOut(stats []*libpod.PodContainerStats) {
+	outFormat := ("%-14s %-14s %-12s %-6s %-19s %-6s %-19s %-19s %-4s\n")
+	fmt.Printf(outFormat, "POD", "CID", "NAME", "CPU %", "MEM USAGE/ LIMIT", "MEM %", "NET IO", "BLOCK IO", "PIDS")
+	for _, i := range stats {
+		if len(i.ContainerStats) == 0 {
+			fmt.Printf(outFormat, i.Pod.ID()[:12], "--", "--", "--", "--", "--", "--", "--", "--")
+		}
+		for _, c := range i.ContainerStats {
+			cpu := floatToPercentString(c.CPU)
+			memUsage := combineHumanValues(c.MemUsage, c.MemLimit)
+			memPerc := floatToPercentString(c.MemPerc)
+			netIO := combineHumanValues(c.NetInput, c.NetOutput)
+			blockIO := combineHumanValues(c.BlockInput, c.BlockOutput)
+			pids := pidsToString(c.PIDs)
+			containerName := c.Name
+			if len(c.Name) > 10 {
+				containerName = containerName[:10]
+			}
+			fmt.Printf(outFormat, i.Pod.ID()[:12], c.ContainerID[:12], containerName, cpu, memUsage, memPerc, netIO, blockIO, pids)
+		}
+	}
+	fmt.Println()
+}
+
+func getPreviousPodContainerStats(podID string, prev []*libpod.PodContainerStats) map[string]*libpod.ContainerStats {
+	for _, p := range prev {
+		if podID == p.Pod.ID() {
+			return p.ContainerStats
+		}
+	}
+	return map[string]*libpod.ContainerStats{}
+}
+
+func outputJson(stats []*libpod.PodContainerStats) error {
+	b, err := json.MarshalIndent(&stats, "", "     ")
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(b))
+	return nil
+}
+
+func getPodsByList(podList []string, r *libpod.Runtime) ([]*libpod.Pod, error) {
+	var (
+		pods []*libpod.Pod
+	)
+	for _, p := range podList {
+		pod, err := r.GetPod(p)
+		if err != nil {
+			return nil, err
+		}
+		pods = append(pods, pod)
+	}
+	return pods, nil
+}

cmd/podman/stats.go

@@ -138,6 +138,10 @@ func statsCmd(c *cli.Context) error {
 			id := ctr.ID()
 			if _, ok := containerStats[ctr.ID()]; !ok {
 				initialStats, err := ctr.GetContainerStats(&libpod.ContainerStats{})
+				if errors.Cause(err) == libpod.ErrCtrRemoved || errors.Cause(err) == libpod.ErrNoSuchCtr {
+					// skip dealing with a container that is gone
+					continue
+				}
 				if err != nil {
 					return err
 				}

libpod/container_ffjson.go

@@ -1,5 +1,5 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: /home/mcs/code/gopath//src/github.com/containers/libpod/libpod/container.go
+// source: libpod/container.go
 
 package libpod
 

libpod/pod.go

@@ -148,16 +148,54 @@ func (p *Pod) AllContainersByID() ([]string, error) {
 
 // AllContainers retrieves the containers in the pod
 func (p *Pod) AllContainers() ([]*Container, error) {
-	p.lock.Lock()
-	defer p.lock.Unlock()
-
 	if !p.valid {
 		return nil, ErrPodRemoved
 	}
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	return p.allContainers()
+}
 
+func (p *Pod) allContainers() ([]*Container, error) {
 	return p.runtime.state.PodContainers(p)
 }
 
 // TODO add pod batching
 // Lock pod to avoid lock contention
 // Store and lock all containers (no RemoveContainer in batch guarantees cache will not become stale)
+
+// PodContainerStats is an organization struct for pods and their containers
+type PodContainerStats struct {
+	Pod            *Pod
+	ContainerStats map[string]*ContainerStats
+}
+
+// GetPodStats returns the stats for each of its containers
+func (p *Pod) GetPodStats(previousContainerStats map[string]*ContainerStats) (map[string]*ContainerStats, error) {
+	var (
+		ok       bool
+		prevStat *ContainerStats
+	)
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	if err := p.updatePod(); err != nil {
+		return nil, err
+	}
+	containers, err := p.runtime.state.PodContainers(p)
+	if err != nil {
+		return nil, err
+	}
+	newContainerStats := make(map[string]*ContainerStats)
+	for _, c := range containers {
+		if prevStat, ok = previousContainerStats[c.ID()]; !ok {
+			prevStat = &ContainerStats{}
+		}
+		newStats, err := c.GetContainerStats(prevStat)
+		if err != nil {
+			return nil, err
+		}
+		newContainerStats[c.ID()] = newStats
+	}
+	return newContainerStats, nil
+}

libpod/pod_ffjson.go

@@ -1,10 +1,11 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: /home/dwalsh/go/src/github.com/containers/libpod/libpod/pod.go
+// source: libpod/pod.go
 
 package libpod
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	fflib "github.com/pquerna/ffjson/fflib/v1"
@@ -849,6 +850,342 @@ done:
 	return nil
 }
 
+// MarshalJSON marshal bytes to json - template
+func (j *PodContainerStats) MarshalJSON() ([]byte, error) {
+	var buf fflib.Buffer
+	if j == nil {
+		buf.WriteString("null")
+		return buf.Bytes(), nil
+	}
+	err := j.MarshalJSONBuf(&buf)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// MarshalJSONBuf marshal buff to json - template
+func (j *PodContainerStats) MarshalJSONBuf(buf fflib.EncodingBuffer) error {
+	if j == nil {
+		buf.WriteString("null")
+		return nil
+	}
+	var err error
+	var obj []byte
+	_ = obj
+	_ = err
+	if j.Pod != nil {
+		/* Struct fall back. type=libpod.Pod kind=struct */
+		buf.WriteString(`{"Pod":`)
+		err = buf.Encode(j.Pod)
+		if err != nil {
+			return err
+		}
+	} else {
+		buf.WriteString(`{"Pod":null`)
+	}
+	buf.WriteString(`,"ContainerStats":`)
+	/* Falling back. type=map[string]*libpod.ContainerStats kind=map */
+	err = buf.Encode(j.ContainerStats)
+	if err != nil {
+		return err
+	}
+	buf.WriteByte('}')
+	return nil
+}
+
+const (
+	ffjtPodContainerStatsbase = iota
+	ffjtPodContainerStatsnosuchkey
+
+	ffjtPodContainerStatsPod
+
+	ffjtPodContainerStatsContainerStats
+)
+
+var ffjKeyPodContainerStatsPod = []byte("Pod")
+
+var ffjKeyPodContainerStatsContainerStats = []byte("ContainerStats")
+
+// UnmarshalJSON umarshall json - template of ffjson
+func (j *PodContainerStats) UnmarshalJSON(input []byte) error {
+	fs := fflib.NewFFLexer(input)
+	return j.UnmarshalJSONFFLexer(fs, fflib.FFParse_map_start)
+}
+
+// UnmarshalJSONFFLexer fast json unmarshall - template ffjson
+func (j *PodContainerStats) UnmarshalJSONFFLexer(fs *fflib.FFLexer, state fflib.FFParseState) error {
+	var err error
+	currentKey := ffjtPodContainerStatsbase
+	_ = currentKey
+	tok := fflib.FFTok_init
+	wantedTok := fflib.FFTok_init
+
+mainparse:
+	for {
+		tok = fs.Scan()
+		//	println(fmt.Sprintf("debug: tok: %v  state: %v", tok, state))
+		if tok == fflib.FFTok_error {
+			goto tokerror
+		}
+
+		switch state {
+
+		case fflib.FFParse_map_start:
+			if tok != fflib.FFTok_left_bracket {
+				wantedTok = fflib.FFTok_left_bracket
+				goto wrongtokenerror
+			}
+			state = fflib.FFParse_want_key
+			continue
+
+		case fflib.FFParse_after_value:
+			if tok == fflib.FFTok_comma {
+				state = fflib.FFParse_want_key
+			} else if tok == fflib.FFTok_right_bracket {
+				goto done
+			} else {
+				wantedTok = fflib.FFTok_comma
+				goto wrongtokenerror
+			}
+
+		case fflib.FFParse_want_key:
+			// json {} ended. goto exit. woo.
+			if tok == fflib.FFTok_right_bracket {
+				goto done
+			}
+			if tok != fflib.FFTok_string {
+				wantedTok = fflib.FFTok_string
+				goto wrongtokenerror
+			}
+
+			kn := fs.Output.Bytes()
+			if len(kn) <= 0 {
+				// "" case. hrm.
+				currentKey = ffjtPodContainerStatsnosuchkey
+				state = fflib.FFParse_want_colon
+				goto mainparse
+			} else {
+				switch kn[0] {
+
+				case 'C':
+
+					if bytes.Equal(ffjKeyPodContainerStatsContainerStats, kn) {
+						currentKey = ffjtPodContainerStatsContainerStats
+						state = fflib.FFParse_want_colon
+						goto mainparse
+					}
+
+				case 'P':
+
+					if bytes.Equal(ffjKeyPodContainerStatsPod, kn) {
+						currentKey = ffjtPodContainerStatsPod
+						state = fflib.FFParse_want_colon
+						goto mainparse
+					}
+
+				}
+
+				if fflib.EqualFoldRight(ffjKeyPodContainerStatsContainerStats, kn) {
+					currentKey = ffjtPodContainerStatsContainerStats
+					state = fflib.FFParse_want_colon
+					goto mainparse
+				}
+
+				if fflib.SimpleLetterEqualFold(ffjKeyPodContainerStatsPod, kn) {
+					currentKey = ffjtPodContainerStatsPod
+					state = fflib.FFParse_want_colon
+					goto mainparse
+				}
+
+				currentKey = ffjtPodContainerStatsnosuchkey
+				state = fflib.FFParse_want_colon
+				goto mainparse
+			}
+
+		case fflib.FFParse_want_colon:
+			if tok != fflib.FFTok_colon {
+				wantedTok = fflib.FFTok_colon
+				goto wrongtokenerror
+			}
+			state = fflib.FFParse_want_value
+			continue
+		case fflib.FFParse_want_value:
+
+			if tok == fflib.FFTok_left_brace || tok == fflib.FFTok_left_bracket || tok == fflib.FFTok_integer || tok == fflib.FFTok_double || tok == fflib.FFTok_string || tok == fflib.FFTok_bool || tok == fflib.FFTok_null {
+				switch currentKey {
+
+				case ffjtPodContainerStatsPod:
+					goto handle_Pod
+
+				case ffjtPodContainerStatsContainerStats:
+					goto handle_ContainerStats
+
+				case ffjtPodContainerStatsnosuchkey:
+					err = fs.SkipField(tok)
+					if err != nil {
+						return fs.WrapErr(err)
+					}
+					state = fflib.FFParse_after_value
+					goto mainparse
+				}
+			} else {
+				goto wantedvalue
+			}
+		}
+	}
+
+handle_Pod:
+
+	/* handler: j.Pod type=libpod.Pod kind=struct quoted=false*/
+
+	{
+		/* Falling back. type=libpod.Pod kind=struct */
+		tbuf, err := fs.CaptureField(tok)
+		if err != nil {
+			return fs.WrapErr(err)
+		}
+
+		err = json.Unmarshal(tbuf, &j.Pod)
+		if err != nil {
+			return fs.WrapErr(err)
+		}
+	}
+
+	state = fflib.FFParse_after_value
+	goto mainparse
+
+handle_ContainerStats:
+
+	/* handler: j.ContainerStats type=map[string]*libpod.ContainerStats kind=map quoted=false*/
+
+	{
+
+		{
+			if tok != fflib.FFTok_left_bracket && tok != fflib.FFTok_null {
+				return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for ", tok))
+			}
+		}
+
+		if tok == fflib.FFTok_null {
+			j.ContainerStats = nil
+		} else {
+
+			j.ContainerStats = make(map[string]*ContainerStats, 0)
+
+			wantVal := true
+
+			for {
+
+				var k string
+
+				var tmpJContainerStats *ContainerStats
+
+				tok = fs.Scan()
+				if tok == fflib.FFTok_error {
+					goto tokerror
+				}
+				if tok == fflib.FFTok_right_bracket {
+					break
+				}
+
+				if tok == fflib.FFTok_comma {
+					if wantVal == true {
+						// TODO(pquerna): this isn't an ideal error message, this handles
+						// things like [,,,] as an array value.
+						return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok))
+					}
+					continue
+				} else {
+					wantVal = true
+				}
+
+				/* handler: k type=string kind=string quoted=false*/
+
+				{
+
+					{
+						if tok != fflib.FFTok_string && tok != fflib.FFTok_null {
+							return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok))
+						}
+					}
+
+					if tok == fflib.FFTok_null {
+
+					} else {
+
+						outBuf := fs.Output.Bytes()
+
+						k = string(string(outBuf))
+
+					}
+				}
+
+				// Expect ':' after key
+				tok = fs.Scan()
+				if tok != fflib.FFTok_colon {
+					return fs.WrapErr(fmt.Errorf("wanted colon token, but got token: %v", tok))
+				}
+
+				tok = fs.Scan()
+				/* handler: tmpJContainerStats type=*libpod.ContainerStats kind=ptr quoted=false*/
+
+				{
+
+					if tok == fflib.FFTok_null {
+						tmpJContainerStats = nil
+					} else {
+						if tmpJContainerStats == nil {
+							tmpJContainerStats = new(ContainerStats)
+						}
+
+						/* handler: tmpJContainerStats type=libpod.ContainerStats kind=struct quoted=false*/
+
+						{
+							/* Falling back. type=libpod.ContainerStats kind=struct */
+							tbuf, err := fs.CaptureField(tok)
+							if err != nil {
+								return fs.WrapErr(err)
+							}
+
+							err = json.Unmarshal(tbuf, &tmpJContainerStats)
+							if err != nil {
+								return fs.WrapErr(err)
+							}
+						}
+
+					}
+				}
+
+				j.ContainerStats[k] = tmpJContainerStats
+
+				wantVal = false
+			}
+
+		}
+	}
+
+	state = fflib.FFParse_after_value
+	goto mainparse
+
+wantedvalue:
+	return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok))
+wrongtokenerror:
+	return fs.WrapErr(fmt.Errorf("ffjson: wanted token: %v, but got token: %v output=%s", wantedTok, tok, fs.Output.String()))
+tokerror:
+	if fs.BigError != nil {
+		return fs.WrapErr(fs.BigError)
+	}
+	err = fs.Error.ToError()
+	if err != nil {
+		return fs.WrapErr(err)
+	}
+	panic("ffjson-generated: unreachable, please report bug.")
+done:
+
+	return nil
+}
+
 // MarshalJSON marshal bytes to json - template
 func (j *PodInspect) MarshalJSON() ([]byte, error) {
 	var buf fflib.Buffer

libpod/runtime_pod.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"time"
 
+	"github.com/containers/libpod/pkg/util"
 	"github.com/pkg/errors"
 )
 
@@ -129,3 +130,36 @@ func (r *Runtime) GetLatestPod() (*Pod, error) {
 	}
 	return pods[lastCreatedIndex], nil
 }
+
+// GetRunningPods returns an array of running pods
+func (r *Runtime) GetRunningPods() ([]*Pod, error) {
+	var (
+		pods        []string
+		runningPods []*Pod
+	)
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	if !r.valid {
+		return nil, ErrRuntimeStopped
+	}
+	containers, err := r.GetRunningContainers()
+	if err != nil {
+		return nil, err
+	}
+	// Assemble running pods
+	for _, c := range containers {
+		if !util.StringInSlice(c.PodID(), pods) {
+			pods = append(pods, c.PodID())
+			pod, err := r.GetPod(c.PodID())
+			if err != nil {
+				if errors.Cause(err) == ErrPodRemoved || errors.Cause(err) == ErrNoSuchPod {
+					continue
+				}
+				return nil, err
+			}
+			runningPods = append(runningPods, pod)
+		}
+	}
+	return runningPods, nil
+}

test/e2e/pod_stats_test.go

@@ -0,0 +1,122 @@
+package integration
+
+import (
+	"fmt"
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Podman pod stats", func() {
+	var (
+		tempdir    string
+		err        error
+		podmanTest PodmanTest
+	)
+
+	BeforeEach(func() {
+		tempdir, err = CreateTempDirInTempDir()
+		if err != nil {
+			os.Exit(1)
+		}
+		podmanTest = PodmanCreate(tempdir)
+		podmanTest.RestoreAllArtifacts()
+	})
+
+	AfterEach(func() {
+		podmanTest.CleanupPod()
+		f := CurrentGinkgoTestDescription()
+		timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds())
+		GinkgoWriter.Write([]byte(timedResult))
+	})
+	It("podman stats should run with no pods", func() {
+		session := podmanTest.Podman([]string{"pod", "stats", "--no-stream"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+	})
+
+	It("podman stats with a bogus pod", func() {
+		session := podmanTest.Podman([]string{"pod", "stats", "foobar"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(125))
+	})
+
+	It("podman stats on a specific running pod", func() {
+		session := podmanTest.Podman([]string{"pod", "create"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		podid := session.OutputToString()
+
+		session = podmanTest.RunTopContainerInPod("", podid)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.RunTopContainerInPod("", podid)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		stats := podmanTest.Podman([]string{"pod", "stats", "--no-stream", podid})
+		stats.WaitWithDefaultTimeout()
+		Expect(stats.ExitCode()).To(Equal(0))
+	})
+
+	It("podman stats on running pods", func() {
+		session := podmanTest.Podman([]string{"pod", "create"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		podid := session.OutputToString()
+
+		session = podmanTest.RunTopContainerInPod("", podid)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.RunTopContainerInPod("", podid)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		stats := podmanTest.Podman([]string{"pod", "stats", "--no-stream"})
+		stats.WaitWithDefaultTimeout()
+		Expect(stats.ExitCode()).To(Equal(0))
+	})
+
+	It("podman stats on all pods", func() {
+		session := podmanTest.Podman([]string{"pod", "create"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		podid := session.OutputToString()
+
+		session = podmanTest.RunTopContainerInPod("", podid)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.RunTopContainerInPod("", podid)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		stats := podmanTest.Podman([]string{"pod", "stats", "--no-stream", "-a"})
+		stats.WaitWithDefaultTimeout()
+		Expect(stats.ExitCode()).To(Equal(0))
+	})
+
+	It("podman stats with json output", func() {
+		session := podmanTest.Podman([]string{"pod", "create"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		podid := session.OutputToString()
+
+		session = podmanTest.RunTopContainerInPod("", podid)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.RunTopContainerInPod("", podid)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		stats := podmanTest.Podman([]string{"pod", "stats", "--format", "json", "--no-stream", "-a"})
+		stats.WaitWithDefaultTimeout()
+		Expect(stats.ExitCode()).To(Equal(0))
+		Expect(stats.IsJSONOutputValid()).To(BeTrue())
+	})
+
+})