The system test write with ginkgo

The tests can be filter by --focus and --skip to fit different test
target. Also be able to set global options and cmd options by export
it to ENV to fit different test matrix.

Signed-off-by: Yiqiao Pu <ypu@redhat.com>

Makefile

@@ -31,7 +31,7 @@ BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions
 OCIUMOUNTINSTALLDIR=$(PREFIX)/share/oci-umount/oci-umount.d
 
 SELINUXOPT ?= $(shell test -x /usr/sbin/selinuxenabled && selinuxenabled && echo -Z)
-PACKAGES ?= $(shell $(GO) list -tags "${BUILDTAGS}" ./... | grep -v github.com/containers/libpod/vendor | grep -v e2e )
+PACKAGES ?= $(shell $(GO) list -tags "${BUILDTAGS}" ./... | grep -v github.com/containers/libpod/vendor | grep -v e2e | grep -v system )
 
 COMMIT_NO ?= $(shell git rev-parse HEAD 2> /dev/null || true)
 GIT_COMMIT ?= $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}")
@@ -178,6 +178,9 @@ ginkgo:
 
 localintegration: varlink_generate test-binaries clientintegration ginkgo
 
+localsystem:
+	ginkgo -v -noColor test/system/
+
 clientintegration:
 	$(MAKE) -C contrib/python/podman integration
 	$(MAKE) -C contrib/python/pypodman integration

test/README.md

@@ -100,3 +100,21 @@ make shell
 ```
 
 This will run a container and give you a shell and you can follow the instructions above.
+
+# System test
+System tests are used for testing the *podman* CLI in the context of a complete system. It
+requires that *podman*, all dependencies, and configurations are in place.  The intention of
+system testing is to match as closely as possible with real-world user/developer use-cases
+and environments. The orchestration of the environments and tests is left to external
+tooling.
+
+* `PodmanTestSystem`: System test *struct* as a composite of `PodmanTest`. It will not add any
+options to the command by default. When you run system test, you can set GLOBALOPTIONS,
+PODMAN_SUBCMD_OPTIONS or PODMAN_BINARY in ENV to run the test suite for different test matrices.
+
+## Run system test
+You can run the test with following command:
+
+```
+make localsystem
+```

test/system/libpod_suite_test.go

@@ -0,0 +1,217 @@
+package system
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	. "github.com/containers/libpod/test/utils"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	PODMAN_BINARY string
+	GLOBALOPTIONS = []string{"--cgroup-manager",
+		"--cni-config-dir",
+		"--config", "-c",
+		"--conmon",
+		"--cpu-profile",
+		"--log-level",
+		"--root",
+		"--tmpdir",
+		"--runroot",
+		"--runtime",
+		"--storage-driver",
+		"--storage-opt",
+		"--syslog",
+	}
+	PODMAN_SUBCMD = []string{"attach",
+		"commit",
+		"container",
+		"build",
+		"create",
+		"diff",
+		"exec",
+		"export",
+		"history",
+		"image",
+		"images",
+		"import",
+		"info",
+		"inspect",
+		"kill",
+		"load",
+		"login",
+		"logout",
+		"logs",
+		"mount",
+		"pause",
+		"ps",
+		"pod",
+		"port",
+		"pull",
+		"push",
+		"restart",
+		"rm",
+		"rmi",
+		"run",
+		"save",
+		"search",
+		"start",
+		"stats",
+		"stop",
+		"tag",
+		"top",
+		"umount",
+		"unpause",
+		"version",
+		"wait",
+		"h",
+	}
+	INTEGRATION_ROOT   string
+	ARTIFACT_DIR       = "/tmp/.artifacts"
+	ALPINE             = "docker.io/library/alpine:latest"
+	BB                 = "docker.io/library/busybox:latest"
+	BB_GLIBC           = "docker.io/library/busybox:glibc"
+	fedoraMinimal      = "registry.fedoraproject.org/fedora-minimal:latest"
+	nginx              = "quay.io/baude/alpine_nginx:latest"
+	redis              = "docker.io/library/redis:alpine"
+	registry           = "docker.io/library/registry:2"
+	infra              = "k8s.gcr.io/pause:3.1"
+	defaultWaitTimeout = 90
+)
+
+// PodmanTestSystem struct for command line options
+type PodmanTestSystem struct {
+	PodmanTest
+	GlobalOptions    map[string]string
+	PodmanCmdOptions map[string][]string
+}
+
+// TestLibpod ginkgo master function
+func TestLibpod(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Libpod Suite")
+}
+
+var _ = BeforeSuite(func() {
+})
+
+// PodmanTestCreate creates a PodmanTestSystem instance for the tests
+func PodmanTestCreate(tempDir string) *PodmanTestSystem {
+	var envKey string
+	globalOptions := make(map[string]string)
+	podmanCmdOptions := make(map[string][]string)
+
+	for _, n := range GLOBALOPTIONS {
+		envKey = strings.Replace(strings.ToUpper(strings.Trim(n, "-")), "-", "_", -1)
+		if isEnvSet(envKey) {
+			globalOptions[n] = os.Getenv(envKey)
+		}
+	}
+
+	for _, n := range PODMAN_SUBCMD {
+		envKey = strings.Replace("PODMAN_SUBCMD_OPTIONS", "SUBCMD", strings.ToUpper(n), -1)
+		if isEnvSet(envKey) {
+			podmanCmdOptions[n] = strings.Split(os.Getenv(envKey), " ")
+		}
+	}
+
+	podmanBinary := "podman"
+	if os.Getenv("PODMAN_BINARY") != "" {
+		podmanBinary = os.Getenv("PODMAN_BINARY")
+	}
+
+	p := &PodmanTestSystem{
+		PodmanTest: PodmanTest{
+			PodmanBinary: podmanBinary,
+			ArtifactPath: ARTIFACT_DIR,
+			TempDir:      tempDir,
+		},
+		GlobalOptions:    globalOptions,
+		PodmanCmdOptions: podmanCmdOptions,
+	}
+
+	p.PodmanMakeOptions = p.makeOptions
+
+	return p
+}
+
+func (p *PodmanTestSystem) Podman(args []string) *PodmanSession {
+	return p.PodmanBase(args)
+}
+
+//MakeOptions assembles all the podman options
+func (p *PodmanTestSystem) makeOptions(args []string) []string {
+	var addOptions, subArgs []string
+	for _, n := range GLOBALOPTIONS {
+		if p.GlobalOptions[n] != "" {
+			addOptions = append(addOptions, n, p.GlobalOptions[n])
+		}
+	}
+
+	if len(args) == 0 {
+		return addOptions
+	}
+
+	subCmd := args[0]
+	addOptions = append(addOptions, subCmd)
+	if subCmd == "unmount" {
+		subCmd = "umount"
+	}
+	if subCmd == "help" {
+		subCmd = "h"
+	}
+
+	if _, ok := p.PodmanCmdOptions[subCmd]; ok {
+		m := make(map[string]bool)
+		subArgs = p.PodmanCmdOptions[subCmd]
+		for i := 0; i < len(subArgs); i++ {
+			m[subArgs[i]] = true
+		}
+		for i := 1; i < len(args); i++ {
+			if _, ok := m[args[i]]; !ok {
+				subArgs = append(subArgs, args[i])
+			}
+		}
+	} else {
+		subArgs = args[1:]
+	}
+
+	addOptions = append(addOptions, subArgs...)
+
+	return addOptions
+}
+
+// Cleanup cleans up the temporary store
+func (p *PodmanTestSystem) Cleanup() {
+	// Remove all containers
+	stopall := p.Podman([]string{"stop", "-a", "--timeout", "0"})
+	stopall.WaitWithDefaultTimeout()
+
+	session := p.Podman([]string{"rm", "-fa"})
+	session.Wait(90)
+	// Nuke tempdir
+	if err := os.RemoveAll(p.TempDir); err != nil {
+		fmt.Printf("%q\n", err)
+	}
+}
+
+// CleanupPod cleans up the temporary store
+func (p *PodmanTestSystem) CleanupPod() {
+	// Remove all containers
+	session := p.Podman([]string{"pod", "rm", "-fa"})
+	session.Wait(90)
+	// Nuke tempdir
+	if err := os.RemoveAll(p.TempDir); err != nil {
+		fmt.Printf("%q\n", err)
+	}
+}
+
+// Check if the key is set in Env
+func isEnvSet(key string) bool {
+	_, set := os.LookupEnv(key)
+	return set
+}

test/system/version_test.go

@@ -0,0 +1,51 @@
+package system
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+
+	. "github.com/containers/libpod/test/utils"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Podman version test", func() {
+	var (
+		tempdir    string
+		err        error
+		podmanTest *PodmanTestSystem
+	)
+
+	BeforeEach(func() {
+		tempdir, err = CreateTempDirInTempDir()
+		if err != nil {
+			os.Exit(1)
+		}
+		podmanTest = PodmanTestCreate(tempdir)
+	})
+
+	AfterEach(func() {
+		podmanTest.Cleanup()
+		f := CurrentGinkgoTestDescription()
+		timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds())
+		GinkgoWriter.Write([]byte(timedResult))
+	})
+
+	It("Smoking test: podman version with extra args", func() {
+		logc := podmanTest.Podman([]string{"version", "anything", "-", "--"})
+		logc.WaitWithDefaultTimeout()
+		Expect(logc.ExitCode()).To(Equal(0))
+		ver := logc.OutputToString()
+		Expect(regexp.MatchString("Version:.*?Go Version:.*?OS/Arch", ver)).To(BeTrue())
+	})
+
+	It("Negative test: podman version with extra flag", func() {
+		logc := podmanTest.Podman([]string{"version", "--foo"})
+		logc.WaitWithDefaultTimeout()
+		Expect(logc.ExitCode()).NotTo(Equal(0))
+		err, _ := logc.GrepString("Incorrect Usage: flag provided but not defined: -foo")
+		Expect(err).To(BeTrue())
+	})
+
+})

test/utils/utils.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"runtime"
 	"strings"
 	"time"