Implement review feedback

- document a recommended convention for fail-fast tests

- document the requirement for jq. (And, add a fail-fast
  test for its presence; remove the duplicated checks
  in subtests)

- add further sanity checks to 'help' test. Add missing
  documentation. Remove a no-longer-needed workaround for
  usage-message bug fixed in #2486

- add a documented TEMPLATE

- and, since we're at 1.1, enable 'Remote API' check in
  version test

- better diagnostics in setup/teardown; add vim filetype hint;
  better formatting of actual-vs-expect errors

- new pod-top, logs, build tests

- improve error messages

- add $IMAGE alias for ridiculous $PODMAN_TEST_IMAGE_FQN

- final cleanup, in prep for merge

Signed-off-by: Ed Santiago <santiago@redhat.com>
This commit is contained in:
Ed Santiago
2019-03-05 09:58:30 -07:00
parent 681eae9bcc
commit 589248d2f3
15 changed files with 343 additions and 55 deletions

114
test/system/000-TEMPLATE Normal file
View File

@ -0,0 +1,114 @@
#!/usr/bin/env bats -*- bats -*-
#
# FIXME: short description of the purpose of this module
#
# FIXME: copy this file to 'NNN-yourtestname.bats' and edit as needed.
#
load helpers
@test "podman subcmd - description of this particular test" {
args="some sort of argument list"
run_podman subcmd $args
is "$output" "what we expect" "output from 'podman subcmd $args'"
}
# vim: filetype=sh
###############################################################################
#
# FIXME FIXME FIXME: Most of the time you can cut from here on down.
# FIXME FIXME FIXME: The above template is probably enough for many tests.
# FIXME FIXME FIXME:
# FIXME FIXME FIXME: If you need anything more complicated, read on.
#
# FIXME: This is a bloated test template. It provides mostly stuff for you
# FIXME: to remove, plus stuff for you to base your tests on.
# FIXME:
# FIXME: copy this file to 'NNN-yourtestname.bats' and edit as needed.
# FIXME: Read all FIXMEs, act on them as needed, then remove them.
# FIXME: test w/ $ PODMAN=./bin/podman bats test/system/NNN-yourtestname.bats
#
load helpers
# FIXME: DELETE THESE LINES UNLESS YOU ABSOLUTELY NEED THEM.
# FIXME: Most tests will not need a custom setup/teardown: they are
# FIXME: provided by helpers.bash.
# FIXME: But if you have to do anything special, these give you the
# FIXME: names of the standard setup/teardown so you can call them
# FIXME: before or after your own additions.
function setup() {
basic_setup
# FIXME: you almost certainly want to do your own setup _after_ basic.
}
function teardown() {
# FIXME: you almost certainly want to do your own teardown _before_ basic.
basic_teardown
}
# FIXME: very basic one-pass example
@test "podman FOO - description of test" {
# FIXME: please try to remove this line; that is, try to write tests
# that will pass as both root and rootless.
skip_if_rootless
# FIXME: template for run commands. Always use 'run_podman'!
# FIXME: The '?' means 'ignore exit status'; use a number if you
# FIXME: expect a precise nonzero code, or omit for 0 (usual case).
# FIXME: NEVER EVER RUN 'podman' DIRECTLY. See helpers.bash for why.
run_podman '?' run -d $IMAGE sh -c 'prep..; echo READY'
cid="$output"
wait_for_ready $cid
run_podman logs $cid
# FIXME: example of dprint. This will trigger if PODMAN_TEST_DEBUG=FOO
# FIXME: ...or anything that matches the name assigned in the @test line.
dprint "podman logs $cid -> '$output'"
is "$output" "what are we expecting?" "description of this check"
# Clean up
run_podman rm $cid
}
# FIXME: another example, this time with a test table loop
@test "podman FOO - json - template for playing with json output" {
# FIXME: Define a multiline string in tabular form, using '|' as separator.
# FIXME: Each row defines one test. Each column (there may be as many as
# FIXME: you want) is one field. In the case below we have two, a
# FIXME: json field descriptor and an expected value.
tests="
id | [0-9a-f]\\\{64\\\}
created | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z
size | -\\\?[0-9]\\\+
"
# FIXME: Run a basic podman command. We'll check $output multiple times
# FIXME: in the while loop below.
run_podman history --format json $IMAGE
# FIXME: parse_table is what does all the work, giving us test cases.
parse_table "$tests" | while read field expect; do
# FIXME: this shows a drawback of BATS and bash: we can't include '|'
# FIXME: in the table, but we need to because some images don't
# FIXME: have a CID. So, yeah, this is ugly -- but rare.
if [ "$field" = "id" ]; then expect="$expect\|<missing>";fi
# output is an array of dicts; check each one
count=$(echo "$output" | jq '. | length')
i=0
while [ $i -lt $count ]; do
actual=$(echo "$output" | jq -r ".[$i].$field")
# FIXME: please be sure to note the third field!
# FIXME: that's the test name. Make it something useful! Include
# FIXME: loop variables whenever possible. Don't just say "my test"
is "$actual" "$expect\$" "jq .[$i].$field"
i=$(expr $i + 1)
done
done
}
# vim: filetype=sh

View File

@ -14,16 +14,13 @@ function setup() {
run_podman version
is "${lines[0]}" "Version:[ ]\+[1-9][0-9.]\+" "Version line 1"
is "$output" ".*Go Version: \+" "'Go Version' in output"
# FIXME: enable for 1.1
# is "$output" ".*RemoteAPI Version: \+" "API version in output"
is "$output" ".*RemoteAPI Version: \+" "API version in output"
}
@test "podman can pull an image" {
run_podman pull $PODMAN_TEST_IMAGE_FQN
run_podman pull $IMAGE
}
# This is for development only; it's intended to make sure our timeout
@ -33,6 +30,21 @@ function setup() {
if [ -z "$PODMAN_RUN_TIMEOUT_TEST" ]; then
skip "define \$PODMAN_RUN_TIMEOUT_TEST to enable this test"
fi
PODMAN_TIMEOUT=10 run_podman run $PODMAN_TEST_IMAGE_FQN sleep 90
PODMAN_TIMEOUT=10 run_podman run $IMAGE sleep 90
echo "*** SHOULD NEVER GET HERE"
}
# Too many tests rely on jq for parsing JSON.
#
# If absolutely necessary, one could establish a convention such as
# defining PODMAN_TEST_SKIP_JQ=1 and adding a skip_if_no_jq() helper.
# For now, let's assume this is not absolutely necessary.
@test "jq is installed and produces reasonable output" {
type -path jq >/dev/null || die "FATAL: 'jq' tool not found."
run jq -r .a.b < <(echo '{ "a": { "b" : "you found me" } }')
is "$output" "you found me" "sample invocation of 'jq'"
}
# vim: filetype=sh

View File

@ -26,8 +26,6 @@ RunRoot:
}
@test "podman info - json" {
type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'"
run_podman info --format=json
expr_nvr="[a-z0-9-]\\\+-[a-z0-9.]\\\+-[a-z0-9]\\\+\."
@ -52,3 +50,5 @@ store.ImageStore.number | 1
done
}
# vim: filetype=sh

View File

@ -25,8 +25,6 @@ load helpers
@test "podman images - json" {
type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'"
tests="
names[0] | $PODMAN_TEST_IMAGE_FQN
id | [0-9a-f]\\\{64\\\}
@ -44,3 +42,5 @@ size | [0-9]\\\+
done
}
# vim: filetype=sh

View File

@ -24,23 +24,26 @@ function podman_commands() {
function check_help() {
count=0
local count=0
local subcommands_found=0
for cmd in $(podman_commands "$@"); do
dprint "podman $@ $cmd --help"
run_podman "$@" $cmd --help
# FIXME FIXME FIXME
usage=$(echo "$output" | grep -A2 '^Usage:' | grep . | tail -1)
# dprint "$usage"
# The line immediately after 'Usage:' gives us a 1-line synopsis
usage=$(echo "$output" | grep -A1 '^Usage:' | tail -1)
[ -n "$usage" ] || die "podman $cmd: no Usage message found"
# if ends in '[command]', recurse into subcommands
# If usage ends in '[command]', recurse into subcommands
if expr "$usage" : '.*\[command\]$' >/dev/null; then
subcommands_found=$(expr $subcommands_found + 1)
check_help "$@" $cmd
continue
fi
# if ends in '[flag]' FIXME
# If usage ends in '[flag]', command takes no more arguments.
# Confirm that by running with 'invalid-arg' and expecting failure.
if expr "$usage" : '.*\[flags\]$' >/dev/null; then
if [ "$cmd" != "help" ]; then
run_podman 125 "$@" $cmd invalid-arg
@ -52,11 +55,22 @@ function check_help() {
count=$(expr $count + 1)
done
# This can happen if the output of --help changes, such as between
# the old command parser and cobra.
[ $count -gt 0 ] || \
die "Internal error: no commands found in 'podman help $@' list"
# At least the top level must have some subcommands
if [ -z "$*" -a $subcommands_found -eq 0 ]; then
die "Internal error: did not find any podman subcommands"
fi
}
@test "podman help - basic tests" {
# Called with no args -- start with 'podman --help'. check_help() will
# recurse for any subcommands.
check_help
}
# vim: filetype=sh

View File

@ -26,7 +26,9 @@ echo $rand | 0 | $rand
# a way to do so.
eval set "$cmd"
run_podman $expected_rc run $PODMAN_TEST_IMAGE_FQN "$@"
run_podman $expected_rc run $IMAGE "$@"
is "$output" "$expected_output" "podman run $cmd - output"
done < <(parse_table "$tests")
}
# vim: filetype=sh

24
test/system/035-logs.bats Normal file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env bats -*- bats -*-
#
# Basic tests for podman logs
#
load helpers
@test "podman logs - basic test" {
rand_string=$(random_string 40)
run_podman create $IMAGE echo $rand_string
cid="$output"
run_podman logs $cid
is "$output" "" "logs on created container: empty"
run_podman start --attach --interactive $cid
is "$output" "$rand_string" "output from podman-start on created ctr"
is "$output" "$rand_string" "logs of started container"
run_podman rm $cid
}
# vim: filetype=sh

View File

@ -5,20 +5,20 @@ load helpers
@test "podman ps - basic tests" {
rand_name=$(random_string 30)
run_podman run -d --name $rand_name $PODMAN_TEST_IMAGE_FQN sleep 5
run_podman run -d --name $rand_name $IMAGE sleep 5
cid=$output
is "$cid" "[0-9a-f]\{64\}$"
# Special case: formatted ps
run_podman ps --no-trunc \
--format '{{.ID}} {{.Image}} {{.Command}} {{.Names}}'
is "$output" "$cid $PODMAN_TEST_IMAGE_FQN sleep 5 $rand_name" "podman ps"
is "$output" "$cid $IMAGE sleep 5 $rand_name" "podman ps"
# Plain old regular ps
run_podman ps
is "${lines[1]}" \
"${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN \+sleep [0-9]\+ .*second.* $cname"\
"${cid:0:12} \+$IMAGE \+sleep [0-9]\+ .*second.* $cname"\
"output from podman ps"
# OK. Wait for sleep to finish...
@ -27,10 +27,12 @@ load helpers
# ...then make sure container shows up as stopped
run_podman ps -a
is "${lines[1]}" \
"${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN *sleep .* Exited .* $rand_name" \
"${cid:0:12} \+$IMAGE *sleep .* Exited .* $rand_name" \
"podman ps -a"
run_podman rm $cid
}
# vim: filetype=sh

View File

@ -4,7 +4,7 @@ load helpers
# Very simple test
@test "podman stop - basic test" {
run_podman run -d $PODMAN_TEST_IMAGE_FQN sleep 60
run_podman run -d $IMAGE sleep 60
cid="$output"
# Run 'stop'. Time how long it takes.
@ -37,7 +37,7 @@ load helpers
# different variations of this test.
for t_opt in '' '--time=5' '--timeout=5'; do
# Run a simple container that logs output on SIGTERM
run_podman run -d $PODMAN_TEST_IMAGE_FQN sh -c \
run_podman run -d $IMAGE sh -c \
"trap 'echo Received SIGTERM, finishing; exit' SIGTERM; echo READY; while :; do sleep 1; done"
cid="$output"
wait_for_ready $cid
@ -63,3 +63,5 @@ load helpers
run_podman rm $cid
done
}
# vim: filetype=sh

View File

@ -11,7 +11,7 @@ load helpers
f_content=$(random_string 30)
c_name=mount_test_$(random_string 5)
run_podman run --name $c_name $PODMAN_TEST_IMAGE_FQN \
run_podman run --name $c_name $IMAGE \
sh -c "echo $f_content > $f_path"
run_podman mount $c_name
@ -33,3 +33,5 @@ load helpers
die "Mounted file exists even after umount: $mount_path/$f_path"
fi
}
# vim: filetype=sh

View File

@ -0,0 +1,28 @@
#!/usr/bin/env bats -*- bats -*-
#
# Tests for podman build
#
load helpers
@test "podman build - basic test" {
rand_filename=$(random_string 20)
rand_content=$(random_string 50)
tmpdir=$PODMAN_TMPDIR/build-test
run mkdir -p $tmpdir || die "Could not mkdir $tmpdir"
dockerfile=$tmpdir/Dockerfile
cat >$dockerfile <<EOF
FROM $IMAGE
RUN echo $rand_content > /$rand_filename
EOF
run_podman build -t build_test --format=docker $tmpdir
run_podman run --rm build_test cat /$rand_filename
is "$output" "$rand_content" "reading generated file in image"
run_podman rmi build_test
}
# vim: filetype=sh

View File

@ -16,21 +16,19 @@ load helpers
eval set -- "$options"
run_podman history "$@" $PODMAN_TEST_IMAGE_FQN
run_podman history "$@" $IMAGE
is "$output" "$expect" "podman history $options"
done
}
@test "podman history - json" {
type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'"
tests="
id | [0-9a-f]\\\{64\\\}
created | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z
size | -\\\?[0-9]\\\+
"
run_podman history --format json $PODMAN_TEST_IMAGE_FQN
run_podman history --format json $IMAGE
parse_table "$tests" | while read field expect; do
# HACK: we can't include '|' in the table
@ -47,3 +45,5 @@ size | -\\\?[0-9]\\\+
done
}
# vim: filetype=sh

View File

@ -0,0 +1,37 @@
#!/usr/bin/env bats
load helpers
@test "podman pod top - containers in different PID namespaces" {
skip_if_rootless
run_podman pod create
podid="$output"
# Start two containers...
run_podman run -d --pod $podid $IMAGE top -d 2
cid1="$output"
run_podman run -d --pod $podid $IMAGE top -d 2
cid2="$output"
# ...and wait for them to actually start.
wait_for_output "PID \+PPID \+USER " $cid1
wait_for_output "PID \+PPID \+USER " $cid2
# Both containers have emitted at least one top-like line.
# Now run 'pod top', and expect two 'top -d 2' processes running.
run_podman pod top $podid
is "$output" ".*root.*top -d 2.*root.*top -d 2" "two 'top' containers"
# There should be a /pause container
# FIXME: sometimes there is, sometimes there isn't. If anyone ever
# actually figures this out, please either reenable this line or
# remove it entirely.
#is "$output" ".*0 \+1 \+0 \+[0-9. ?s]\+/pause" "there is a /pause container"
# Clean up
run_podman pod rm -f $podid
}
# vim: filetype=sh

View File

@ -18,17 +18,28 @@ sometimes needs to massage the returned values; `015-run.bats` offers
examples of how to deal with the more typical such issues.
* `run_podman` - runs command defined in `$PODMAN` (default: 'podman'
but could also be 'podman-remote'), with a timeout. Checks its exit status.
but could also be './bin/podman' or 'podman-remote'), with a timeout.
Checks its exit status.
* `is` - compare actual vs expected output. Emits a useful diagnostic
on failure.
* `die` - output a properly-formatted message to stderr, and fail test
* `skip_if_rootless` - if rootless, skip this test with a helpful message.
* `random_string` - returns a pseudorandom alphanumeric string
Test files are of the form `NNN-name.bats` where NNN is a three-digit
number. Please preserve this convention, it simplifies viewing the
directory and understanding test order. Most of the time it's not
important but `00x` should be reserved for the times when it matters.
directory and understanding test order. In particular, `00x` tests
should be reserved for a first-pass fail-fast subset of tests:
bats test/system/00*.bats || exit 1
bats test/system
...the goal being to provide quick feedback on catastrophic failures
without having to wait for the entire test suite.
Analyzing test failures
@ -56,6 +67,12 @@ set `PODMAN_TEST_DEBUG="funcname"` where `funcname` is the name of
the function or perhaps just a substring.
Requirements
============
The `jq` tool is needed for parsing JSON output.
Further Details
===============

View File

@ -10,6 +10,9 @@ PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"alpine_labels"}
PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"latest"}
PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG"
# Because who wants to spell that out each time?
IMAGE=$PODMAN_TEST_IMAGE_FQN
# Default timeout for a podman command.
PODMAN_TIMEOUT=${PODMAN_TIMEOUT:-60}
@ -33,9 +36,9 @@ function basic_setup() {
if [ "$1" == "$PODMAN_TEST_IMAGE_FQN" ]; then
found_needed_image=1
else
echo "# setup_standard_environment: podman rmi $1 & $2" >&3
podman rmi --force "$1" >/dev/null 2>&1 || true
podman rmi --force "$2" >/dev/null 2>&1 || true
echo "# setup(): removing stray images" >&3
run_podman rmi --force "$1" >/dev/null 2>&1 || true
run_podman rmi --force "$2" >/dev/null 2>&1 || true
fi
done
@ -43,11 +46,22 @@ function basic_setup() {
if [ -z "$found_needed_image" ]; then
run_podman pull "$PODMAN_TEST_IMAGE_FQN"
fi
# Argh. Although BATS provides $BATS_TMPDIR, it's just /tmp!
# That's bloody worthless. Let's make our own, in which subtests
# can write whatever they like and trust that it'll be deleted
# on cleanup.
# TODO: do this outside of setup, so it carries across tests?
PODMAN_TMPDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-/tmp} podman_bats.XXXXXX)
}
# Basic teardown: remove all containers
# Basic teardown: remove all pods and containers
function basic_teardown() {
run_podman rm --all --force
echo "# [teardown]" >&2
run_podman '?' pod rm --all --force
run_podman '?' rm --all --force
/bin/rm -rf $PODMAN_TMPDIR
}
@ -118,10 +132,12 @@ function run_podman() {
esac
# stdout is only emitted upon error; this echo is to help a debugger
echo "\$ $PODMAN $@"
echo "\$ $PODMAN $*"
run timeout --foreground -v --kill=10 $PODMAN_TIMEOUT $PODMAN "$@"
# without "quotes", multiple lines are glommed together into one
echo "$output"
if [ -n "$output" ]; then
echo "$output"
fi
if [ "$status" -ne 0 ]; then
echo -n "[ rc=$status ";
if [ -n "$expected_rc" ]; then
@ -143,21 +159,22 @@ function run_podman() {
if [ -n "$expected_rc" ]; then
if [ "$status" -ne "$expected_rc" ]; then
die "FAIL: exit code is $status; expected $expected_rc"
die "exit code is $status; expected $expected_rc"
fi
fi
}
# Wait for 'READY' in container output
function wait_for_ready {
local cid=
# Wait for certain output from a container, indicating that it's ready.
function wait_for_output {
local sleep_delay=5
local how_long=60
local how_long=$PODMAN_TIMEOUT
local expect=
local cid=
# Arg processing. A single-digit number is how long to sleep between
# iterations; a 2- or 3-digit number is the total time to wait; anything
# else is the container ID or name to wait on.
# iterations; a 2- or 3-digit number is the total time to wait; all
# else are, in order, the string to expect and the container name/ID.
local i
for i in "$@"; do
if expr "$i" : '[0-9]\+$' >/dev/null; then
@ -166,6 +183,8 @@ function wait_for_ready {
else
how_long=$i
fi
elif [ -z "$expect" ]; then
expect=$i
else
cid=$i
fi
@ -176,14 +195,19 @@ function wait_for_ready {
t1=$(expr $SECONDS + $how_long)
while [ $SECONDS -lt $t1 ]; do
run_podman logs $cid
if expr "$output" : ".*READY" >/dev/null; then
if expr "$output" : ".*$expect" >/dev/null; then
return
fi
sleep $sleep_delay
done
die "FAIL: timed out waiting for READY from $cid"
die "timed out waiting for '$expect' from $cid"
}
# Shortcut for the lazy
function wait_for_ready {
wait_for_output 'READY' "$@"
}
# END podman helpers
@ -206,7 +230,9 @@ function skip_if_rootless() {
# die # Abort with helpful message
#########
function die() {
echo "# $*" >&2
echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" >&2
echo "#| FAIL: $*" >&2
echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2
false
}
@ -232,16 +258,24 @@ function is() {
if [ -z "$actual" ]; then
return
fi
die "$testname:\n# expected no output; got %q\n" "$actual"
fi
if expr "$actual" : "$expect" >/dev/null; then
expect='[no output]'
elif expr "$actual" : "$expect" >/dev/null; then
return
fi
# This is a multi-line message, so let's format it ourself (not via die)
printf "# $testname:\n# expected: %q\n# actual: %q\n" \
"$expect" "$actual" >&2
# This is a multi-line message, which may in turn contain multi-line
# output, so let's format it ourself, readably
local -a actual_split
readarray -t actual_split <<<"$actual"
printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2
printf "#| FAIL: $testname\n" >&2
printf "#| expected: '%s'\n" "$expect" >&2
printf "#| actual: '%s'\n" "${actual_split[0]}" >&2
local line
for line in "${actual_split[@]:1}"; do
printf "#| > '%s'\n" "$line" >&2
done
printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2
false
}