new system tests under BATS

Initial attempt at writing a framework for podman system tests.
The idea is to define a useful set of primitives that will
make it easy to write actual tests and to interpret results
of failing ones.

This is a proof-of-concept right now; only a small number of
tests, by no means comprehensive. I am requesting review in
order to find showstopper problems: reasons why this approach
cannot work. Should there be none, we can work toward running
these as gating tests for Fedora and RHEL8.

Signed-off-by: Ed Santiago <santiago@redhat.com>

test/system/001-basic.bats

@@ -0,0 +1,38 @@
+#!/usr/bin/env bats
+#
+# Simplest set of podman tests. If any of these fail, we have serious problems.
+#
+
+load helpers
+
+# Override standard setup! We don't yet trust podman-images or podman-rm
+function setup() {
+    :
+}
+
+@test "podman version emits reasonable output" {
+    run_podman version
+
+    is "${lines[0]}" "Version:[ ]\+[1-9][0-9.]\+" "Version line 1"
+
+    is "$output" ".*Go Version: \+"               "'Go Version' in output"
+
+    # FIXME: enable for 1.1
+#    is "$output" ".*RemoteAPI Version: \+"        "API version in output"
+}
+
+
+@test "podman can pull an image" {
+    run_podman pull $PODMAN_TEST_IMAGE_FQN
+}
+
+# This is for development only; it's intended to make sure our timeout
+# in run_podman continues to work. This test should never run in production
+# because it will, by definition, fail.
+@test "timeout" {
+    if [ -z "$PODMAN_RUN_TIMEOUT_TEST" ]; then
+        skip "define \$PODMAN_RUN_TIMEOUT_TEST to enable this test"
+    fi
+    PODMAN_TIMEOUT=10 run_podman run $PODMAN_TEST_IMAGE_FQN sleep 90
+    echo "*** SHOULD NEVER GET HERE"
+}

test/system/005-info.bats

@@ -0,0 +1,54 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "podman info - basic test" {
+    run_podman info
+
+    expected_keys="
+BuildahVersion: *[0-9.]\\\+
+Conmon:\\\s\\\+package:
+Distribution:
+OCIRuntime:\\\s\\\+package:
+os:
+rootless:
+insecure registries:
+store:
+GraphDriverName:
+GraphRoot:
+GraphStatus:
+ImageStore:\\\s\\\+number: 1
+RunRoot:
+"
+    while read expect; do
+        is "$output" ".*$expect" "output includes '$expect'"
+    done < <(parse_table "$expected_keys")
+}
+
+@test "podman info - json" {
+    type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'"
+
+    run_podman info --format=json
+
+    expr_nvr="[a-z0-9-]\\\+-[a-z0-9.]\\\+-[a-z0-9]\\\+\."
+    expr_path="/[a-z0-9\\\/.]\\\+\\\$"
+
+    tests="
+host.BuildahVersion       | [0-9.]
+host.Conmon.package       | $expr_nvr
+host.Conmon.path          | $expr_path
+host.OCIRuntime.package   | $expr_nvr
+host.OCIRuntime.path      | $expr_path
+store.ConfigFile          | $expr_path
+store.GraphDriverName     | [a-z0-9]\\\+\\\$
+store.GraphRoot           | $expr_path
+store.ImageStore.number   | 1
+"
+
+    parse_table "$tests" | while read field expect; do
+        actual=$(echo "$output" | jq -r ".$field")
+        dprint "# actual=<$actual> expect=<$expect>"
+        is "$actual" "$expect" "jq .$field"
+    done
+
+}

test/system/010-images.bats

@@ -0,0 +1,46 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "podman images - basic output" {
+    run_podman images -a
+
+    is "${lines[0]}" "REPOSITORY *TAG *IMAGE ID *CREATED *SIZE" "header line"
+    is "${lines[1]}" "$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME *$PODMAN_TEST_IMAGE_TAG *[0-9a-f]\+" "podman images output"
+}
+
+@test "podman images - custom formats" {
+    tests="
+--format {{.ID}}                  |        [0-9a-f]\\\{12\\\}
+--format {{.ID}} --no-trunc       | sha256:[0-9a-f]\\\{64\\\}
+--format {{.Repository}}:{{.Tag}} | $PODMAN_TEST_IMAGE_FQN
+"
+
+    parse_table "$tests" | while read fmt expect; do
+        run_podman images $fmt
+        is "$output" "$expect\$" "podman images $fmt"
+    done
+
+}
+
+
+@test "podman images - json" {
+    type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'"
+
+    tests="
+names[0]   | $PODMAN_TEST_IMAGE_FQN
+id         |        [0-9a-f]\\\{64\\\}
+digest     | sha256:[0-9a-f]\\\{64\\\}
+created    | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z
+size       | [0-9]\\\+
+"
+
+    run_podman images -a --format json
+
+    parse_table "$tests" | while read field expect; do
+        actual=$(echo "$output" | jq -r ".[0].$field")
+        dprint "# actual=<$actual> expect=<$expect}>"
+        is "$actual" "$expect" "jq .$field"
+    done
+
+}

test/system/015-help.bats

@@ -0,0 +1,62 @@
+#!/usr/bin/env bats
+#
+# Tests based on 'podman help'
+#
+# Find all commands listed by 'podman --help'. Run each one, make sure it
+# provides its own --help output. If the usage message ends in '[command]',
+# treat it as a subcommand, and recurse into its own list of sub-subcommands.
+#
+# Any usage message that ends in '[flags]' is interpreted as a command
+# that takes no further arguments; we confirm by running with 'invalid-arg'
+# and confirming that it exits with error status and message.
+#
+load helpers
+
+# run 'podman help', parse the output looking for 'Available Commands';
+# return that list.
+function podman_commands() {
+    dprint "$@"
+    run_podman help "$@" |\
+        awk '/^Available Commands:/{ok=1;next}/^Flags:/{ok=0}ok { print $1 }' |\
+        grep .
+    "$output"
+}
+
+
+function check_help() {
+    count=0
+    for cmd in $(podman_commands "$@"); do
+        dprint "podman $@ $cmd --help"
+        run_podman "$@" $cmd --help
+
+        # FIXME FIXME FIXME
+        usage=$(echo "$output" | grep -A2 '^Usage:' | grep . | tail -1)
+        #        dprint "$usage"
+        [ -n "$usage" ] || die "podman $cmd: no Usage message found"
+
+        # if ends in '[command]', recurse into subcommands
+        if expr "$usage" : '.*\[command\]$' >/dev/null; then
+            check_help "$@" $cmd
+            continue
+        fi
+
+        # if ends in '[flag]' FIXME
+        if expr "$usage" : '.*\[flags\]$' >/dev/null; then
+            if [ "$cmd" != "help" ]; then
+                run_podman 125 "$@" $cmd invalid-arg
+                is "$output" "Error: .* takes no arguments" \
+                   "'podman $@ $cmd' with extra (invalid) arguments"
+            fi
+        fi
+
+        count=$(expr $count + 1)
+    done
+
+    [ $count -gt 0 ] || \
+        die "Internal error: no commands found in 'podman help $@' list"
+}
+
+
+@test "podman help - basic tests" {
+    check_help
+}

test/system/030-run.bats

@@ -0,0 +1,32 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "podman run - basic tests" {
+    rand=$(random_string 30)
+    tests="
+true              |   0 |
+false             |   1 |
+sh -c 'exit 32'   |  32 |
+echo $rand        |   0 | $rand
+/no/such/command  | 127 | Error: container create failed:.*exec:.* no such file or dir
+/etc              | 126 | Error: container create failed:.*exec:.* permission denied
+"
+
+    while read cmd expected_rc expected_output; do
+        if [ "$expected_output" = "''" ]; then expected_output=""; fi
+
+        # THIS IS TRICKY: this is what lets us handle a quoted command.
+        # Without this incantation (and the "$@" below), the cmd string
+        # gets passed on as individual tokens: eg "sh" "-c" "'exit" "32'"
+        # (note unmatched opening and closing single-quotes in the last 2).
+        # That results in a bizarre and hard-to-understand failure
+        # in the BATS 'run' invocation.
+        # This should really be done inside parse_table; I can't find
+        # a way to do so.
+        eval set "$cmd"
+
+        run_podman $expected_rc run $PODMAN_TEST_IMAGE_FQN "$@"
+        is "$output" "$expected_output" "podman run $cmd - output"
+    done < <(parse_table "$tests")
+}

test/system/040-ps.bats

@@ -0,0 +1,36 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "podman ps - basic tests" {
+    rand_name=$(random_string 30)
+
+    run_podman run -d --name $rand_name $PODMAN_TEST_IMAGE_FQN sleep 5
+    cid=$output
+    is "$cid" "[0-9a-f]\{64\}$"
+
+    # Special case: formatted ps
+    run_podman ps --no-trunc \
+               --format '{{.ID}} {{.Image}} {{.Command}} {{.Names}}'
+    is "$output" "$cid $PODMAN_TEST_IMAGE_FQN sleep 5 $rand_name" "podman ps"
+
+
+    # Plain old regular ps
+    run_podman ps
+    is "${lines[1]}" \
+       "${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN \+sleep [0-9]\+ .*second.* $cname"\
+       "output from podman ps"
+
+    # OK. Wait for sleep to finish...
+    run_podman wait $cid
+
+    # ...then make sure container shows up as stopped
+    run_podman ps -a
+    is "${lines[1]}" \
+       "${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN *sleep .* Exited .* $rand_name" \
+       "podman ps -a"
+
+
+
+    run_podman rm $cid
+}

test/system/050-stop.bats

@@ -0,0 +1,65 @@
+#!/usr/bin/env bats
+
+load helpers
+
+# Very simple test
+@test "podman stop - basic test" {
+    run_podman run -d $PODMAN_TEST_IMAGE_FQN sleep 60
+    cid="$output"
+
+    # Run 'stop'. Time how long it takes.
+    t0=$SECONDS
+    run_podman stop $cid
+    t1=$SECONDS
+
+    # Confirm that container is stopped
+    run_podman inspect --format '{{.State.Status}} {{.State.ExitCode}}' $cid
+    is "$output" "exited \+137" "Status and exit code of stopped container"
+
+    # The initial SIGTERM is ignored, so this operation should take
+    # exactly 10 seconds. Give it some leeway.
+    delta_t=$(( $t1 - $t0 ))
+    [ $delta_t -gt 8 ]  ||\
+        die "podman stop: ran too quickly! ($delta_t seconds; expected >= 10)"
+    [ $delta_t -le 14 ] ||\
+        die "podman stop: took too long ($delta_t seconds; expected ~10)"
+
+    run_podman rm $cid
+}
+
+
+# Test fallback
+
+
+# Regression test for #2472
+@test "podman stop - can trap signal" {
+    # Because the --time and --timeout options can be wonky, try three
+    # different variations of this test.
+    for t_opt in '' '--time=5' '--timeout=5'; do
+        # Run a simple container that logs output on SIGTERM
+        run_podman run -d $PODMAN_TEST_IMAGE_FQN sh -c \
+                   "trap 'echo Received SIGTERM, finishing; exit' SIGTERM; echo READY; while :; do sleep 1; done"
+        cid="$output"
+        wait_for_ready $cid
+
+        # Run 'stop' against it...
+        t0=$SECONDS
+        run_podman stop $t_opt $cid
+        t1=$SECONDS
+
+        # ...the container should trap the signal, log it, and exit.
+        run_podman logs $cid
+        is "$output" ".*READY.*Received SIGTERM, finishing" "podman stop $t_opt"
+
+        # Exit code should be 0, because container did its own exit
+        run_podman inspect --format '{{.State.ExitCode}}' $cid
+        is "$output" "0" "Exit code of stopped container"
+
+        # The 'stop' command should return almost instantaneously
+        delta_t=$(( $t1 - $t0 ))
+        [ $delta_t -le 2 ] ||\
+            die "podman stop: took too long ($delta_t seconds; expected <= 2)"
+
+        run_podman rm $cid
+    done
+}

test/system/060-mount.bats

@@ -0,0 +1,35 @@
+#!/usr/bin/env bats
+
+load helpers
+
+
+@test "podman mount - basic test" {
+    # Only works with root (FIXME: does it work with rootless + vfs?)
+    skip_if_rootless
+
+    f_path=/tmp/tmpfile_$(random_string 8)
+    f_content=$(random_string 30)
+
+    c_name=mount_test_$(random_string 5)
+    run_podman run --name $c_name $PODMAN_TEST_IMAGE_FQN \
+               sh -c "echo $f_content > $f_path"
+
+    run_podman mount $c_name
+    mount_path=$output
+
+    test -d $mount_path
+    test -e "$mount_path/$f_path"
+    is $(< "$mount_path/$f_path") "$f_content" "contents of file, as read via fs"
+
+    # Make sure that 'podman mount' (no args) returns the expected path
+    run_podman mount --notruncate
+    # FIXME: is it worth the effort to validate the CID ($1) ?
+    reported_mountpoint=$(echo "$output" | awk '{print $2}')
+    is $reported_mountpoint $mount_path "mountpoint reported by 'podman mount'"
+
+    # umount, and make sure files are gone
+    run_podman umount $c_name
+    if [ -e "$mount_path/$f_path" ]; then
+        die "Mounted file exists even after umount: $mount_path/$f_path"
+    fi
+}

test/system/110-history.bats

@@ -0,0 +1,49 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "podman history - basic tests" {
+    tests="
+                                 | .*[0-9a-f]\\\{12\\\} .* CMD .* LABEL
+--format '{{.ID}} {{.Created}}'  | .*[0-9a-f]\\\{12\\\} .* ago
+--human=false                    | .*[0-9a-f]\\\{12\\\} *[0-9-]\\\+T[0-9:]\\\+Z
+-qH                              | .*[0-9a-f]\\\{12\\\}
+--no-trunc                       | .*[0-9a-f]\\\{64\\\}
+"
+
+    parse_table "$tests" | while read options expect; do
+        if [ "$options" = "''" ]; then options=; fi
+
+        eval set -- "$options"
+
+        run_podman history "$@" $PODMAN_TEST_IMAGE_FQN
+        is "$output" "$expect" "podman history $options"
+    done
+}
+
+@test "podman history - json" {
+    type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'"
+
+    tests="
+id        | [0-9a-f]\\\{64\\\}
+created   | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z
+size      | -\\\?[0-9]\\\+
+"
+
+    run_podman history --format json $PODMAN_TEST_IMAGE_FQN
+
+    parse_table "$tests" | while read field expect; do
+        # HACK: we can't include '|' in the table
+        if [ "$field" = "id" ]; then expect="$expect\|<missing>";fi
+
+        # output is an array of dicts; check each one
+        count=$(echo "$output" | jq '. | length')
+        i=0
+        while [ $i -lt $count ]; do
+            actual=$(echo "$output" | jq -r ".[$i].$field")
+            is "$actual" "$expect\$" "jq .[$i].$field"
+            i=$(expr $i + 1)
+        done
+    done
+
+}

test/system/README.md

@@ -0,0 +1,65 @@
+Quick overview of podman system tests. The idea is to use BATS,
+but with a framework for making it easy to add new tests and to
+debug failures.
+
+Quick Start
+===========
+
+Look at [030-run.bats](030-run.bats) for a simple but packed example.
+This introduces the basic set of helper functions:
+
+* `setup` (implicit) - resets container storage so there's
+one and only one (standard) image, and no running containers.
+
+* `parse_table` - you can define tables of inputs and expected results,
+then read those in a `while` loop. This makes it easy to add new tests.
+Because bash is not a programming language, the caller of `parse_table`
+sometimes needs to massage the returned values; `015-run.bats` offers
+examples of how to deal with the more typical such issues.
+
+* `run_podman` - runs command defined in `$PODMAN` (default: 'podman'
+but could also be 'podman-remote'), with a timeout. Checks its exit status.
+
+* `is` - compare actual vs expected output. Emits a useful diagnostic
+on failure.
+
+* `random_string` - returns a pseudorandom alphanumeric string
+
+Test files are of the form `NNN-name.bats` where NNN is a three-digit
+number. Please preserve this convention, it simplifies viewing the
+directory and understanding test order. Most of the time it's not
+important but `00x` should be reserved for the times when it matters.
+
+
+Analyzing test failures
+=======================
+
+The top priority for this scheme is to make it easy to diagnose
+what went wrong. To that end, `podman_run` always logs all invoked
+commands, their output and exit codes. In a normal run you will never
+see this, but BATS will display it on failure. The goal here is to
+give you everything you need to diagnose without having to rerun tests.
+
+The `is` comparison function is designed to emit useful diagnostics,
+in particular, the actual and expected strings. Please do not use
+the horrible BATS standard of `[ x = y ]`; that's nearly useless
+for tracking down failures.
+
+If the above are not enough to help you track down a failure:
+
+
+Debugging tests
+---------------
+
+Some functions have `dprint` statements. To see the output of these,
+set `PODMAN_TEST_DEBUG="funcname"` where `funcname` is the name of
+the function or perhaps just a substring.
+
+
+Further Details
+===============
+
+TBD. For now, look in [helpers.bash](helpers.bash); each helper function
+has (what are intended to be) helpful header comments. For even more
+examples, see and/or run `helpers.t`; that's a regression test
+and provides a thorough set of examples of how the helpers work.

test/system/helpers.bash

@@ -0,0 +1,315 @@
+# -*- bash -*-
+
+# Podman command to run; may be podman-remote
+PODMAN=${PODMAN:-podman}
+
+# Standard image to use for most tests
+PODMAN_TEST_IMAGE_REGISTRY=${PODMAN_TEST_IMAGE_REGISTRY:-"quay.io"}
+PODMAN_TEST_IMAGE_USER=${PODMAN_TEST_IMAGE_USER:-"libpod"}
+PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"alpine_labels"}
+PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"latest"}
+PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG"
+
+# Default timeout for a podman command.
+PODMAN_TIMEOUT=${PODMAN_TIMEOUT:-60}
+
+###############################################################################
+# BEGIN setup/teardown tools
+
+# Provide common setup and teardown functions, but do not name them such!
+# That way individual tests can override with their own setup/teardown,
+# while retaining the ability to include these if they so desire.
+
+# Setup helper: establish a test environment with exactly the images needed
+function basic_setup() {
+    # Clean up all containers
+    run_podman rm --all --force
+
+    # Clean up all images except those desired
+    found_needed_image=
+    run_podman images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}'
+    for line in "${lines[@]}"; do
+        set $line
+        if [ "$1" == "$PODMAN_TEST_IMAGE_FQN" ]; then
+            found_needed_image=1
+        else
+            echo "# setup_standard_environment: podman rmi $1 & $2" >&3
+            podman rmi --force "$1" >/dev/null 2>&1 || true
+            podman rmi --force "$2" >/dev/null 2>&1 || true
+        fi
+    done
+
+    # Make sure desired images are present
+    if [ -z "$found_needed_image" ]; then
+        run_podman pull "$PODMAN_TEST_IMAGE_FQN"
+    fi
+}
+
+# Basic teardown: remove all containers
+function basic_teardown() {
+    run_podman rm --all --force
+}
+
+
+# Provide the above as default methods.
+function setup() {
+    basic_setup
+}
+
+function teardown() {
+    basic_teardown
+}
+
+
+# Helpers useful for tests running rmi
+function archive_image() {
+    local image=$1
+
+    # FIXME: refactor?
+    archive_basename=$(echo $1 | tr -c a-zA-Z0-9._- _)
+    archive=$BATS_TMPDIR/$archive_basename.tar
+
+    run_podman save -o $archive $image
+}
+
+function restore_image() {
+    local image=$1
+
+    archive_basename=$(echo $1 | tr -c a-zA-Z0-9._- _)
+    archive=$BATS_TMPDIR/$archive_basename.tar
+
+    run_podman restore $archive
+}
+
+# END   setup/teardown tools
+###############################################################################
+# BEGIN podman helpers
+
+################
+#  run_podman  #  Invoke $PODMAN, with timeout, using BATS 'run'
+################
+#
+# This is the preferred mechanism for invoking podman: first, it
+# invokes $PODMAN, which may be 'podman-remote' or '/some/path/podman'.
+#
+# Second, we use 'timeout' to abort (with a diagnostic) if something
+# takes too long; this is preferable to a CI hang.
+#
+# Third, we log the command run and its output. This doesn't normally
+# appear in BATS output, but it will if there's an error.
+#
+# Next, we check exit status. Since the normal desired code is 0,
+# that's the default; but the first argument can override:
+#
+#     run_podman 125  nonexistent-subcommand
+#     run_podman '?'  some-other-command       # let our caller check status
+#
+# Since we use the BATS 'run' mechanism, $output and $status will be
+# defined for our caller.
+#
+function run_podman() {
+    # Number as first argument = expected exit code; default 0
+    expected_rc=0
+    case "$1" in
+        [0-9])           expected_rc=$1; shift;;
+        [1-9][0-9])      expected_rc=$1; shift;;
+        [12][0-9][0-9])  expected_rc=$1; shift;;
+        '?')             expected_rc=  ; shift;;  # ignore exit code
+    esac
+
+    # stdout is only emitted upon error; this echo is to help a debugger
+    echo "\$ $PODMAN $@"
+    run timeout --foreground -v --kill=10 $PODMAN_TIMEOUT $PODMAN "$@"
+    # without "quotes", multiple lines are glommed together into one
+    echo "$output"
+    if [ "$status" -ne 0 ]; then
+        echo -n "[ rc=$status ";
+        if [ -n "$expected_rc" ]; then
+            if [ "$status" -eq "$expected_rc" ]; then
+                echo -n "(expected) ";
+            else
+                echo -n "(** EXPECTED $expected_rc **) ";
+            fi
+        fi
+        echo "]"
+    fi
+
+    if [ "$status" -eq 124 ]; then
+        if expr "$output" : ".*timeout: sending" >/dev/null; then
+            echo "*** TIMED OUT ***"
+            false
+        fi
+    fi
+
+    if [ -n "$expected_rc" ]; then
+        if [ "$status" -ne "$expected_rc" ]; then
+            die "FAIL: exit code is $status; expected $expected_rc"
+        fi
+    fi
+}
+
+
+# Wait for 'READY' in container output
+function wait_for_ready {
+    local cid=
+    local sleep_delay=5
+    local how_long=60
+
+    # Arg processing. A single-digit number is how long to sleep between
+    # iterations; a 2- or 3-digit number is the total time to wait; anything
+    # else is the container ID or name to wait on.
+    local i
+    for i in "$@"; do
+        if expr "$i" : '[0-9]\+$' >/dev/null; then
+            if [ $i -le 9 ]; then
+                sleep_delay=$i
+            else
+                how_long=$i
+            fi
+        else
+            cid=$i
+        fi
+    done
+
+    [ -n "$cid" ] || die "FATAL: wait_for_ready: no container name/ID in '$*'"
+
+    t1=$(expr $SECONDS + $how_long)
+    while [ $SECONDS -lt $t1 ]; do
+        run_podman logs $cid
+        if expr "$output" : ".*READY" >/dev/null; then
+            return
+        fi
+
+        sleep $sleep_delay
+    done
+
+    die "FAIL: timed out waiting for READY from $cid"
+}
+
+# END   podman helpers
+###############################################################################
+# BEGIN miscellaneous tools
+
+######################
+#  skip_if_rootless  #  ...with an optional message
+######################
+function skip_if_rootless() {
+    if [ "$(id -u)" -eq 0 ]; then
+        return
+    fi
+
+    skip "${1:-not applicable under rootless podman}"
+}
+
+
+#########
+#  die  #  Abort with helpful message
+#########
+function die() {
+    echo "# $*" >&2
+    false
+}
+
+
+########
+#  is  #  Compare actual vs expected string; fail w/diagnostic if mismatch
+########
+#
+# Compares given string against expectations, using 'expr' to allow patterns.
+#
+# Examples:
+#
+#   is "$actual" "$expected" "descriptive test name"
+#   is "apple" "orange"  "name of a test that will fail in most universes"
+#   is "apple" "[a-z]\+" "this time it should pass"
+#
+function is() {
+    local actual="$1"
+    local expect="$2"
+    local testname="${3:-FIXME}"
+
+    if [ -z "$expect" ]; then
+        if [ -z "$actual" ]; then
+            return
+        fi
+        die "$testname:\n#  expected no output; got %q\n" "$actual"
+    fi
+
+    if expr "$actual" : "$expect" >/dev/null; then
+        return
+    fi
+
+    # This is a multi-line message, so let's format it ourself (not via die)
+    printf "# $testname:\n#  expected: %q\n#    actual: %q\n"    \
+           "$expect" "$actual" >&2
+    false
+}
+
+
+############
+#  dprint  #  conditional debug message
+############
+#
+# Set PODMAN_TEST_DEBUG to the name of one or more functions you want to debug
+#
+# Examples:
+#
+#    $ PODMAN_TEST_DEBUG=parse_table bats .
+#    $ PODMAN_TEST_DEBUG="test_podman_images test_podman_run" bats .
+#
+function dprint() {
+    test -z "$PODMAN_TEST_DEBUG" && return
+
+    caller="${FUNCNAME[1]}"
+
+    # PODMAN_TEST_DEBUG is a space-separated list of desired functions
+    # e.g. "parse_table test_podman_images" (or even just "table")
+    for want in $PODMAN_TEST_DEBUG; do
+        # Check if our calling function matches any of the desired strings
+        if expr "$caller" : ".*$want" >/dev/null; then
+            echo "# ${FUNCNAME[1]}() : $*" >&3
+            return
+        fi
+    done
+}
+
+
+#################
+#  parse_table  #  Split a table on '|' delimiters; return space-separated
+#################
+#
+# See sample .bats scripts for examples. The idea is to list a set of
+# tests in a table, then use simple logic to iterate over each test.
+# Columns are separated using '|' (pipe character) because sometimes
+# we need spaces in our fields.
+#
+function parse_table() {
+    while read line; do
+        test -z "$line" && continue
+
+        declare -a row=()
+        while read col; do
+            dprint "col=<<$col>>"
+            row+=("$col")
+        done <  <(echo "$line" | tr '|' '\012' | sed -e 's/^ *//' -e 's/\\/\\\\/g')
+
+        printf "%q " "${row[@]}"
+        printf "\n"
+    done <<<"$1"
+}
+
+
+###################
+#  random_string  #  Returns a pseudorandom human-readable string
+###################
+#
+# Numeric argument, if present, is desired length of string
+#
+function random_string() {
+    local length=${1:-10}
+
+    head /dev/urandom | tr -dc a-zA-Z0-9 | head -c$length
+}
+
+# END   miscellaneous tools
+###############################################################################

test/system/helpers.t

@@ -0,0 +1,145 @@
+#!/bin/bash
+#
+# regression tests for helpers.bash
+#
+# Some of those helper functions are fragile, and we don't want to break
+# anything if we have to mess with them.
+#
+
+source $(dirname $0)/helpers.bash
+
+die() {
+    echo "$(basename $0): $*" >&2
+    exit 1
+}
+
+# Iterator and return code; updated in check_result()
+testnum=0
+rc=0
+
+###############################################################################
+# BEGIN test the parse_table helper
+
+function check_result {
+    testnum=$(expr $testnum + 1)
+    if [ "$1" = "$2" ]; then
+        echo "ok $testnum $3 = $1"
+    else
+        echo "not ok $testnum $3"
+        echo "#  expected: $2"
+        echo "#    actual: $1"
+        rc=1
+    fi
+}
+
+# IMPORTANT NOTE: you have to do
+#      this: while ... done < <(parse_table)
+#   and not: parse_table | while read ...
+#
+# ...because piping to 'while' makes it a subshell, hence testnum and rc
+# will not be updated.
+#
+while read x y z; do
+    check_result "$x" "a" "parse_table simple: column 1"
+    check_result "$y" "b" "parse_table simple: column 2"
+    check_result "$z" "c" "parse_table simple: column 3"
+done < <(parse_table "a | b | c")
+
+# More complicated example, with spaces
+while read x y z; do
+    check_result "$x" "a b"   "parse_table with spaces: column 1"
+    check_result "$y" "c d"   "parse_table with spaces: column 2"
+    check_result "$z" "e f g" "parse_table with spaces: column 3"
+done < <(parse_table "a b | c d | e f g")
+
+# Multi-row, with spaces and with blank lines
+table="
+a     | b   | c d e
+d e f | g h | i j
+"
+declare -A expect=(
+    [0,0]="a"
+    [0,1]="b"
+    [0,2]="c d e"
+    [1,0]="d e f"
+    [1,1]="g h"
+    [1,2]="i j"
+)
+row=0
+while read x y z;do
+    check_result "$x" "${expect[$row,0]}" "parse_table multi_row[$row,0]"
+    check_result "$y" "${expect[$row,1]}" "parse_table multi_row[$row,1]"
+    check_result "$z" "${expect[$row,2]}" "parse_table multi_row[$row,2]"
+    row=$(expr $row + 1)
+done < <(parse_table "$table")
+
+# Backslash handling. The first element should have none, the second some
+while read x y;do
+    check_result "$x" '[0-9]{2}'    "backslash test - no backslashes"
+    check_result "$y" '[0-9]\{3\}'  "backslash test - one backslash each"
+done < <(parse_table "[0-9]{2}  | [0-9]\\\{3\\\}")
+
+# Empty strings. I wish we could convert those to real empty strings.
+while read x y z; do
+    check_result "$x" "''" "empty string - left-hand"
+    check_result "$y" "''" "empty string - middle"
+    check_result "$z" "''" "empty string - right"
+done < <(parse_table " | |")
+
+# Quotes
+while read x y z;do
+    check_result "$x" "a 'b c'"     "single quotes"
+    check_result "$y" "d \"e f\" g" "double quotes"
+    check_result "$z" "h"           "no quotes"
+
+    # FIXME FIXME FIXME: this is the only way I can find to get bash-like
+    # splitting of tokens. It really should be done inside parse_table
+    # but I can't find any way of doing so. If you can find a way, please
+    # update this test and any BATS tests that rely on quoting.
+    eval set "$x"
+    check_result "$1" "a"     "single quotes - token split - 1"
+    check_result "$2" "b c"   "single quotes - token split - 2"
+    check_result "$3" ""      "single quotes - token split - 3"
+
+    eval set "$y"
+    check_result "$1" "d"     "double quotes - token split - 1"
+    check_result "$2" "e f"   "double quotes - token split - 2"
+    check_result "$3" "g"     "double quotes - token split - 3"
+done < <(parse_table "a 'b c' | d \"e f\" g | h")
+
+# END   test the parse_table helper
+###############################################################################
+# BEGIN dprint
+
+function dprint_test_1() {
+    dprint "$*"
+}
+
+# parse_table works, might as well use it
+#
+#  <value of PODMAN_TEST_DEBUG> | <blank for no msg, - for msg> | <desc>
+#
+table="
+                           |   | debug unset
+dprint_test                | - | substring match
+dprint_test_1              | - | exact match
+dprint_test_10             |   | caller name mismatch
+xxx yyy zzz                |   | multiple callers, no match
+dprint_test_1 xxx yyy zzz  | - | multiple callers, match at start
+xxx dprint_test_1 yyy zzz  | - | multiple callers, match in middle
+xxx yyy zzz dprint_test_1  | - | multiple callers, match at end
+"
+while read var expect name; do
+    random_string=$(random_string 20)
+    PODMAN_TEST_DEBUG="$var" result=$(dprint_test_1 "$random_string" 3>&1)
+    expect_full=""
+    if [ -n "$expect" -a "$expect" != "''" ]; then
+        expect_full="# dprint_test_1() : $random_string"
+    fi
+    check_result "$result" "$expect_full" "DEBUG='$var' - $name"
+done < <(parse_table "$table")
+
+# END   dprint
+###############################################################################
+
+exit $rc

test/system/libpod_suite_test.go

@@ -1,217 +0,0 @@
-package system
-
-import (
-	"fmt"
-	"os"
-	"strings"
-	"testing"
-
-	. "github.com/containers/libpod/test/utils"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var (
-	PODMAN_BINARY string
-	GLOBALOPTIONS = []string{"--cgroup-manager",
-		"--cni-config-dir",
-		"--config", "-c",
-		"--conmon",
-		"--cpu-profile",
-		"--log-level",
-		"--root",
-		"--tmpdir",
-		"--runroot",
-		"--runtime",
-		"--storage-driver",
-		"--storage-opt",
-		"--syslog",
-	}
-	PODMAN_SUBCMD = []string{"attach",
-		"commit",
-		"container",
-		"build",
-		"create",
-		"diff",
-		"exec",
-		"export",
-		"history",
-		"image",
-		"images",
-		"import",
-		"info",
-		"inspect",
-		"kill",
-		"load",
-		"login",
-		"logout",
-		"logs",
-		"mount",
-		"pause",
-		"ps",
-		"pod",
-		"port",
-		"pull",
-		"push",
-		"restart",
-		"rm",
-		"rmi",
-		"run",
-		"save",
-		"search",
-		"start",
-		"stats",
-		"stop",
-		"tag",
-		"top",
-		"umount",
-		"unpause",
-		"version",
-		"wait",
-		"h",
-	}
-	INTEGRATION_ROOT   string
-	ARTIFACT_DIR       = "/tmp/.artifacts"
-	ALPINE             = "docker.io/library/alpine:latest"
-	BB                 = "docker.io/library/busybox:latest"
-	BB_GLIBC           = "docker.io/library/busybox:glibc"
-	fedoraMinimal      = "registry.fedoraproject.org/fedora-minimal:latest"
-	nginx              = "quay.io/baude/alpine_nginx:latest"
-	redis              = "docker.io/library/redis:alpine"
-	registry           = "docker.io/library/registry:2"
-	infra              = "k8s.gcr.io/pause:3.1"
-	defaultWaitTimeout = 90
-)
-
-// PodmanTestSystem struct for command line options
-type PodmanTestSystem struct {
-	PodmanTest
-	GlobalOptions    map[string]string
-	PodmanCmdOptions map[string][]string
-}
-
-// TestLibpod ginkgo master function
-func TestLibpod(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Libpod Suite")
-}
-
-var _ = BeforeSuite(func() {
-})
-
-// PodmanTestCreate creates a PodmanTestSystem instance for the tests
-func PodmanTestCreate(tempDir string) *PodmanTestSystem {
-	var envKey string
-	globalOptions := make(map[string]string)
-	podmanCmdOptions := make(map[string][]string)
-
-	for _, n := range GLOBALOPTIONS {
-		envKey = strings.Replace(strings.ToUpper(strings.Trim(n, "-")), "-", "_", -1)
-		if isEnvSet(envKey) {
-			globalOptions[n] = os.Getenv(envKey)
-		}
-	}
-
-	for _, n := range PODMAN_SUBCMD {
-		envKey = strings.Replace("PODMAN_SUBCMD_OPTIONS", "SUBCMD", strings.ToUpper(n), -1)
-		if isEnvSet(envKey) {
-			podmanCmdOptions[n] = strings.Split(os.Getenv(envKey), " ")
-		}
-	}
-
-	podmanBinary := "podman"
-	if os.Getenv("PODMAN_BINARY") != "" {
-		podmanBinary = os.Getenv("PODMAN_BINARY")
-	}
-
-	p := &PodmanTestSystem{
-		PodmanTest: PodmanTest{
-			PodmanBinary: podmanBinary,
-			ArtifactPath: ARTIFACT_DIR,
-			TempDir:      tempDir,
-		},
-		GlobalOptions:    globalOptions,
-		PodmanCmdOptions: podmanCmdOptions,
-	}
-
-	p.PodmanMakeOptions = p.makeOptions
-
-	return p
-}
-
-func (p *PodmanTestSystem) Podman(args []string) *PodmanSession {
-	return p.PodmanBase(args)
-}
-
-//MakeOptions assembles all the podman options
-func (p *PodmanTestSystem) makeOptions(args []string) []string {
-	var addOptions, subArgs []string
-	for _, n := range GLOBALOPTIONS {
-		if p.GlobalOptions[n] != "" {
-			addOptions = append(addOptions, n, p.GlobalOptions[n])
-		}
-	}
-
-	if len(args) == 0 {
-		return addOptions
-	}
-
-	subCmd := args[0]
-	addOptions = append(addOptions, subCmd)
-	if subCmd == "unmount" {
-		subCmd = "umount"
-	}
-	if subCmd == "help" {
-		subCmd = "h"
-	}
-
-	if _, ok := p.PodmanCmdOptions[subCmd]; ok {
-		m := make(map[string]bool)
-		subArgs = p.PodmanCmdOptions[subCmd]
-		for i := 0; i < len(subArgs); i++ {
-			m[subArgs[i]] = true
-		}
-		for i := 1; i < len(args); i++ {
-			if _, ok := m[args[i]]; !ok {
-				subArgs = append(subArgs, args[i])
-			}
-		}
-	} else {
-		subArgs = args[1:]
-	}
-
-	addOptions = append(addOptions, subArgs...)
-
-	return addOptions
-}
-
-// Cleanup cleans up the temporary store
-func (p *PodmanTestSystem) Cleanup() {
-	// Remove all containers
-	stopall := p.Podman([]string{"stop", "-a", "--timeout", "0"})
-	stopall.WaitWithDefaultTimeout()
-
-	session := p.Podman([]string{"rm", "-fa"})
-	session.Wait(90)
-	// Nuke tempdir
-	if err := os.RemoveAll(p.TempDir); err != nil {
-		fmt.Printf("%q\n", err)
-	}
-}
-
-// CleanupPod cleans up the temporary store
-func (p *PodmanTestSystem) CleanupPod() {
-	// Remove all containers
-	session := p.Podman([]string{"pod", "rm", "-fa"})
-	session.Wait(90)
-	// Nuke tempdir
-	if err := os.RemoveAll(p.TempDir); err != nil {
-		fmt.Printf("%q\n", err)
-	}
-}
-
-// Check if the key is set in Env
-func isEnvSet(key string) bool {
-	_, set := os.LookupEnv(key)
-	return set
-}

test/system/version_test.go

@@ -1,51 +0,0 @@
-package system
-
-import (
-	"fmt"
-	"os"
-	"regexp"
-
-	. "github.com/containers/libpod/test/utils"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Podman version test", func() {
-	var (
-		tempdir    string
-		err        error
-		podmanTest *PodmanTestSystem
-	)
-
-	BeforeEach(func() {
-		tempdir, err = CreateTempDirInTempDir()
-		if err != nil {
-			os.Exit(1)
-		}
-		podmanTest = PodmanTestCreate(tempdir)
-	})
-
-	AfterEach(func() {
-		podmanTest.Cleanup()
-		f := CurrentGinkgoTestDescription()
-		timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds())
-		GinkgoWriter.Write([]byte(timedResult))
-	})
-
-	It("Smoking test: podman version with extra args", func() {
-		logc := podmanTest.Podman([]string{"version", "anything", "-", "--"})
-		logc.WaitWithDefaultTimeout()
-		Expect(logc.ExitCode()).To(Equal(0))
-		ver := logc.OutputToString()
-		Expect(regexp.MatchString("Version:.*?Go Version:.*?OS/Arch", ver)).To(BeTrue())
-	})
-
-	It("Negative test: podman version with extra flag", func() {
-		logc := podmanTest.Podman([]string{"version", "--foo"})
-		logc.WaitWithDefaultTimeout()
-		Expect(logc.ExitCode()).NotTo(Equal(0))
-		err, _ := logc.GrepString("Incorrect Usage: flag provided but not defined: -foo")
-		Expect(err).To(BeTrue())
-	})
-
-})