Implement review feedback

- document a recommended convention for fail-fast tests

- document the requirement for jq. (And, add a fail-fast
  test for its presence; remove the duplicated checks
  in subtests)

- add further sanity checks to 'help' test. Add missing
  documentation. Remove a no-longer-needed workaround for
  usage-message bug fixed in #2486

- add a documented TEMPLATE

- and, since we're at 1.1, enable 'Remote API' check in
  version test

- better diagnostics in setup/teardown; add vim filetype hint;
  better formatting of actual-vs-expect errors

- new pod-top, logs, build tests

- improve error messages

- add $IMAGE alias for ridiculous $PODMAN_TEST_IMAGE_FQN

- final cleanup, in prep for merge

Signed-off-by: Ed Santiago <santiago@redhat.com>

test/system/000-TEMPLATE

@@ -0,0 +1,114 @@
+#!/usr/bin/env bats   -*- bats -*-
+#
+# FIXME: short description of the purpose of this module
+#
+# FIXME: copy this file to 'NNN-yourtestname.bats' and edit as needed.
+#
+
+load helpers
+
+@test "podman subcmd - description of this particular test" {
+    args="some sort of argument list"
+    run_podman subcmd $args
+    is "$output"   "what we expect"   "output from 'podman subcmd $args'"
+}
+
+# vim: filetype=sh
+
+###############################################################################
+#
+# FIXME FIXME FIXME: Most of the time you can cut from here on down.
+# FIXME FIXME FIXME: The above template is probably enough for many tests.
+# FIXME FIXME FIXME:
+# FIXME FIXME FIXME: If you need anything more complicated, read on.
+#
+# FIXME: This is a bloated test template. It provides mostly stuff for you
+# FIXME: to remove, plus stuff for you to base your tests on.
+# FIXME:
+# FIXME: copy this file to 'NNN-yourtestname.bats' and edit as needed.
+# FIXME: Read all FIXMEs, act on them as needed, then remove them.
+# FIXME: test w/ $ PODMAN=./bin/podman bats test/system/NNN-yourtestname.bats
+#
+
+load helpers
+
+# FIXME: DELETE THESE LINES UNLESS YOU ABSOLUTELY NEED THEM.
+# FIXME: Most tests will not need a custom setup/teardown: they are
+# FIXME: provided by helpers.bash.
+# FIXME: But if you have to do anything special, these give you the
+# FIXME: names of the standard setup/teardown so you can call them
+# FIXME: before or after your own additions.
+function setup() {
+    basic_setup
+    # FIXME: you almost certainly want to do your own setup _after_ basic.
+}
+function teardown() {
+    # FIXME: you almost certainly want to do your own teardown _before_ basic.
+    basic_teardown
+}
+
+
+# FIXME: very basic one-pass example
+@test "podman FOO - description of test" {
+    # FIXME: please try to remove this line; that is, try to write tests
+    # that will pass as both root and rootless.
+    skip_if_rootless
+
+    # FIXME: template for run commands. Always use 'run_podman'!
+    # FIXME: The '?' means 'ignore exit status'; use a number if you
+    # FIXME:    expect a precise nonzero code, or omit for 0 (usual case).
+    # FIXME: NEVER EVER RUN 'podman' DIRECTLY. See helpers.bash for why.
+    run_podman '?' run -d $IMAGE sh -c 'prep..; echo READY'
+    cid="$output"
+    wait_for_ready $cid
+
+    run_podman logs $cid
+    # FIXME: example of dprint. This will trigger if PODMAN_TEST_DEBUG=FOO
+    # FIXME:  ...or anything that matches the name assigned in the @test line.
+    dprint "podman logs $cid -> '$output'"
+    is "$output" "what are we expecting?" "description of this check"
+
+    # Clean up
+    run_podman rm $cid
+}
+
+
+# FIXME: another example, this time with a test table loop
+@test "podman FOO - json - template for playing with json output" {
+    # FIXME: Define a multiline string in tabular form, using '|' as separator.
+    # FIXME: Each row defines one test. Each column (there may be as many as
+    # FIXME: you want) is one field. In the case below we have two, a
+    # FIXME: json field descriptor and an expected value.
+    tests="
+id        | [0-9a-f]\\\{64\\\}
+created   | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z
+size      | -\\\?[0-9]\\\+
+"
+
+    # FIXME: Run a basic podman command. We'll check $output multiple times
+    # FIXME: in the while loop below.
+    run_podman history --format json $IMAGE
+
+    # FIXME: parse_table is what does all the work, giving us test cases.
+    parse_table "$tests" | while read field expect; do
+        # FIXME: this shows a drawback of BATS and bash: we can't include '|'
+        # FIXME: in the table, but we need to because some images don't
+        # FIXME: have a CID. So, yeah, this is ugly -- but rare.
+        if [ "$field" = "id" ]; then expect="$expect\|<missing>";fi
+
+        # output is an array of dicts; check each one
+        count=$(echo "$output" | jq '. | length')
+        i=0
+        while [ $i -lt $count ]; do
+            actual=$(echo "$output" | jq -r ".[$i].$field")
+            # FIXME: please be sure to note the third field!
+            # FIXME: that's the test name. Make it something useful! Include
+            # FIXME: loop variables whenever possible. Don't just say "my test"
+            is "$actual"    "$expect\$"    "jq .[$i].$field"
+            i=$(expr $i + 1)
+        done
+    done
+}
+
+
+# vim: filetype=sh

test/system/001-basic.bats

@@ -14,16 +14,13 @@ function setup() {
     run_podman version
 
     is "${lines[0]}" "Version:[ ]\+[1-9][0-9.]\+" "Version line 1"
-
     is "$output" ".*Go Version: \+"               "'Go Version' in output"
-
+    is "$output" ".*RemoteAPI Version: \+"        "API version in output"
-    # FIXME: enable for 1.1
-#    is "$output" ".*RemoteAPI Version: \+"        "API version in output"
 }
 
 
 @test "podman can pull an image" {
-    run_podman pull $PODMAN_TEST_IMAGE_FQN
+    run_podman pull $IMAGE
 }
 
 # This is for development only; it's intended to make sure our timeout
@@ -33,6 +30,21 @@ function setup() {
     if [ -z "$PODMAN_RUN_TIMEOUT_TEST" ]; then
         skip "define \$PODMAN_RUN_TIMEOUT_TEST to enable this test"
     fi
-    PODMAN_TIMEOUT=10 run_podman run $PODMAN_TEST_IMAGE_FQN sleep 90
+    PODMAN_TIMEOUT=10 run_podman run $IMAGE sleep 90
     echo "*** SHOULD NEVER GET HERE"
 }
+
+
+# Too many tests rely on jq for parsing JSON.
+#
+# If absolutely necessary, one could establish a convention such as
+# defining PODMAN_TEST_SKIP_JQ=1 and adding a skip_if_no_jq() helper.
+# For now, let's assume this is not absolutely necessary.
+@test "jq is installed and produces reasonable output" {
+    type -path jq >/dev/null || die "FATAL: 'jq' tool not found."
+
+    run jq -r .a.b < <(echo '{ "a": { "b" : "you found me" } }')
+    is "$output" "you found me" "sample invocation of 'jq'"
+}
+
+# vim: filetype=sh

test/system/005-info.bats

@@ -26,8 +26,6 @@ RunRoot:
 }
 
 @test "podman info - json" {
-    type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'"
-
     run_podman info --format=json
 
     expr_nvr="[a-z0-9-]\\\+-[a-z0-9.]\\\+-[a-z0-9]\\\+\."
@@ -52,3 +50,5 @@ store.ImageStore.number   | 1
     done
 
 }
+
+# vim: filetype=sh

test/system/010-images.bats

@@ -25,8 +25,6 @@ load helpers
 
 
 @test "podman images - json" {
-    type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'"
-
     tests="
 names[0]   | $PODMAN_TEST_IMAGE_FQN
 id         |        [0-9a-f]\\\{64\\\}
@@ -44,3 +42,5 @@ size       | [0-9]\\\+
     done
 
 }
+
+# vim: filetype=sh

test/system/015-help.bats

@@ -24,23 +24,26 @@ function podman_commands() {
 
 
 function check_help() {
-    count=0
+    local count=0
+    local subcommands_found=0
+
     for cmd in $(podman_commands "$@"); do
         dprint "podman $@ $cmd --help"
         run_podman "$@" $cmd --help
 
-        # FIXME FIXME FIXME
+        # The line immediately after 'Usage:' gives us a 1-line synopsis
-        usage=$(echo "$output" | grep -A2 '^Usage:' | grep . | tail -1)
+        usage=$(echo "$output" | grep -A1 '^Usage:' | tail -1)
-        #        dprint "$usage"
         [ -n "$usage" ] || die "podman $cmd: no Usage message found"
 
-        # if ends in '[command]', recurse into subcommands
+        # If usage ends in '[command]', recurse into subcommands
         if expr "$usage" : '.*\[command\]$' >/dev/null; then
+            subcommands_found=$(expr $subcommands_found + 1)
             check_help "$@" $cmd
             continue
         fi
 
-        # if ends in '[flag]' FIXME
+        # If usage ends in '[flag]', command takes no more arguments.
+        # Confirm that by running with 'invalid-arg' and expecting failure.
         if expr "$usage" : '.*\[flags\]$' >/dev/null; then
             if [ "$cmd" != "help" ]; then
                 run_podman 125 "$@" $cmd invalid-arg
@@ -52,11 +55,22 @@ function check_help() {
         count=$(expr $count + 1)
     done
 
+    # This can happen if the output of --help changes, such as between
+    # the old command parser and cobra.
     [ $count -gt 0 ] || \
         die "Internal error: no commands found in 'podman help $@' list"
+
+    # At least the top level must have some subcommands
+    if [ -z "$*" -a $subcommands_found -eq 0 ]; then
+        die "Internal error: did not find any podman subcommands"
+    fi
 }
 
 
 @test "podman help - basic tests" {
+    # Called with no args -- start with 'podman --help'. check_help() will
+    # recurse for any subcommands.
     check_help
 }
+
+# vim: filetype=sh

test/system/030-run.bats

@@ -26,7 +26,9 @@ echo $rand        |   0 | $rand
         # a way to do so.
         eval set "$cmd"
 
-        run_podman $expected_rc run $PODMAN_TEST_IMAGE_FQN "$@"
+        run_podman $expected_rc run $IMAGE "$@"
         is "$output" "$expected_output" "podman run $cmd - output"
     done < <(parse_table "$tests")
 }
+
+# vim: filetype=sh

test/system/035-logs.bats

@@ -0,0 +1,24 @@
+#!/usr/bin/env bats   -*- bats -*-
+#
+# Basic tests for podman logs
+#
+
+load helpers
+
+@test "podman logs - basic test" {
+    rand_string=$(random_string 40)
+
+    run_podman create $IMAGE echo $rand_string
+    cid="$output"
+
+    run_podman logs $cid
+    is "$output" ""  "logs on created container: empty"
+
+    run_podman start --attach --interactive $cid
+    is "$output" "$rand_string" "output from podman-start on created ctr"
+    is "$output" "$rand_string" "logs of started container"
+
+    run_podman rm $cid
+}
+
+# vim: filetype=sh

test/system/040-ps.bats

@@ -5,20 +5,20 @@ load helpers
 @test "podman ps - basic tests" {
     rand_name=$(random_string 30)
 
-    run_podman run -d --name $rand_name $PODMAN_TEST_IMAGE_FQN sleep 5
+    run_podman run -d --name $rand_name $IMAGE sleep 5
     cid=$output
     is "$cid" "[0-9a-f]\{64\}$"
 
     # Special case: formatted ps
     run_podman ps --no-trunc \
                --format '{{.ID}} {{.Image}} {{.Command}} {{.Names}}'
-    is "$output" "$cid $PODMAN_TEST_IMAGE_FQN sleep 5 $rand_name" "podman ps"
+    is "$output" "$cid $IMAGE sleep 5 $rand_name" "podman ps"
 
 
     # Plain old regular ps
     run_podman ps
     is "${lines[1]}" \
-       "${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN \+sleep [0-9]\+ .*second.* $cname"\
+       "${cid:0:12} \+$IMAGE \+sleep [0-9]\+ .*second.* $cname"\
        "output from podman ps"
 
     # OK. Wait for sleep to finish...
@@ -27,10 +27,12 @@ load helpers
     # ...then make sure container shows up as stopped
     run_podman ps -a
     is "${lines[1]}" \
-       "${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN *sleep .* Exited .* $rand_name" \
+       "${cid:0:12} \+$IMAGE *sleep .* Exited .* $rand_name" \
        "podman ps -a"
 
 
 
     run_podman rm $cid
 }
+
+# vim: filetype=sh

test/system/050-stop.bats

@@ -4,7 +4,7 @@ load helpers
 
 # Very simple test
 @test "podman stop - basic test" {
-    run_podman run -d $PODMAN_TEST_IMAGE_FQN sleep 60
+    run_podman run -d $IMAGE sleep 60
     cid="$output"
 
     # Run 'stop'. Time how long it takes.
@@ -37,7 +37,7 @@ load helpers
     # different variations of this test.
     for t_opt in '' '--time=5' '--timeout=5'; do
         # Run a simple container that logs output on SIGTERM
-        run_podman run -d $PODMAN_TEST_IMAGE_FQN sh -c \
+        run_podman run -d $IMAGE sh -c \
                    "trap 'echo Received SIGTERM, finishing; exit' SIGTERM; echo READY; while :; do sleep 1; done"
         cid="$output"
         wait_for_ready $cid
@@ -63,3 +63,5 @@ load helpers
         run_podman rm $cid
     done
 }
+
+# vim: filetype=sh

test/system/060-mount.bats

@@ -11,7 +11,7 @@ load helpers
     f_content=$(random_string 30)
 
     c_name=mount_test_$(random_string 5)
-    run_podman run --name $c_name $PODMAN_TEST_IMAGE_FQN \
+    run_podman run --name $c_name $IMAGE \
                sh -c "echo $f_content > $f_path"
 
     run_podman mount $c_name
@@ -33,3 +33,5 @@ load helpers
         die "Mounted file exists even after umount: $mount_path/$f_path"
     fi
 }
+
+# vim: filetype=sh

test/system/070-build.bats

@@ -0,0 +1,28 @@
+#!/usr/bin/env bats   -*- bats -*-
+#
+# Tests for podman build
+#
+
+load helpers
+
+@test "podman build - basic test" {
+    rand_filename=$(random_string 20)
+    rand_content=$(random_string 50)
+
+    tmpdir=$PODMAN_TMPDIR/build-test
+    run mkdir -p $tmpdir || die "Could not mkdir $tmpdir"
+    dockerfile=$tmpdir/Dockerfile
+    cat >$dockerfile <<EOF
+FROM $IMAGE
+RUN echo $rand_content > /$rand_filename
+EOF
+
+    run_podman build -t build_test --format=docker $tmpdir
+
+    run_podman run --rm build_test cat /$rand_filename
+    is "$output"   "$rand_content"   "reading generated file in image"
+
+    run_podman rmi build_test
+}
+
+# vim: filetype=sh

test/system/110-history.bats

@@ -16,21 +16,19 @@ load helpers
 
         eval set -- "$options"
 
-        run_podman history "$@" $PODMAN_TEST_IMAGE_FQN
+        run_podman history "$@" $IMAGE
         is "$output" "$expect" "podman history $options"
     done
 }
 
 @test "podman history - json" {
-    type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'"
-
     tests="
 id        | [0-9a-f]\\\{64\\\}
 created   | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z
 size      | -\\\?[0-9]\\\+
 "
 
-    run_podman history --format json $PODMAN_TEST_IMAGE_FQN
+    run_podman history --format json $IMAGE
 
     parse_table "$tests" | while read field expect; do
         # HACK: we can't include '|' in the table
@@ -47,3 +45,5 @@ size      | -\\\?[0-9]\\\+
     done
 
 }
+
+# vim: filetype=sh

test/system/200-pod-top.bats

@@ -0,0 +1,37 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "podman pod top - containers in different PID namespaces" {
+    skip_if_rootless
+
+    run_podman pod create
+    podid="$output"
+
+    # Start two containers...
+    run_podman run -d --pod $podid $IMAGE top -d 2
+    cid1="$output"
+    run_podman run -d --pod $podid $IMAGE top -d 2
+    cid2="$output"
+
+    # ...and wait for them to actually start.
+    wait_for_output "PID \+PPID \+USER " $cid1
+    wait_for_output "PID \+PPID \+USER " $cid2
+
+    # Both containers have emitted at least one top-like line.
+    # Now run 'pod top', and expect two 'top -d 2' processes running.
+    run_podman pod top $podid
+    is "$output" ".*root.*top -d 2.*root.*top -d 2" "two 'top' containers"
+
+    # There should be a /pause container
+    # FIXME: sometimes there is, sometimes there isn't. If anyone ever
+    # actually figures this out, please either reenable this line or
+    # remove it entirely.
+    #is "$output" ".*0 \+1 \+0 \+[0-9. ?s]\+/pause" "there is a /pause container"
+
+    # Clean up
+    run_podman pod rm -f $podid
+}
+
+
+# vim: filetype=sh

test/system/README.md

@@ -18,17 +18,28 @@ sometimes needs to massage the returned values; `015-run.bats` offers
 examples of how to deal with the more typical such issues.
 
 * `run_podman` - runs command defined in `$PODMAN` (default: 'podman'
-but could also be 'podman-remote'), with a timeout. Checks its exit status.
+but could also be './bin/podman' or 'podman-remote'), with a timeout.
+Checks its exit status.
 
 * `is` - compare actual vs expected output. Emits a useful diagnostic
 on failure.
 
+* `die` - output a properly-formatted message to stderr, and fail test
+
+* `skip_if_rootless` - if rootless, skip this test with a helpful message.
+
 * `random_string` - returns a pseudorandom alphanumeric string
 
 Test files are of the form `NNN-name.bats` where NNN is a three-digit
 number. Please preserve this convention, it simplifies viewing the
-directory and understanding test order. Most of the time it's not
+directory and understanding test order. In particular, `00x` tests
-important but `00x` should be reserved for the times when it matters.
+should be reserved for a first-pass fail-fast subset of tests:
+
+    bats test/system/00*.bats || exit 1
+    bats test/system
+
+...the goal being to provide quick feedback on catastrophic failures
+without having to wait for the entire test suite.
 
 
 Analyzing test failures
@@ -56,6 +67,12 @@ set `PODMAN_TEST_DEBUG="funcname"` where `funcname` is the name of
 the function or perhaps just a substring.
 
 
+Requirements
+============
+
+The `jq` tool is needed for parsing JSON output.
+
+
 Further Details
 ===============
 

test/system/helpers.bash

@@ -10,6 +10,9 @@ PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"alpine_labels"}
 PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"latest"}
 PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG"
 
+# Because who wants to spell that out each time?
+IMAGE=$PODMAN_TEST_IMAGE_FQN
+
 # Default timeout for a podman command.
 PODMAN_TIMEOUT=${PODMAN_TIMEOUT:-60}
 
@@ -33,9 +36,9 @@ function basic_setup() {
         if [ "$1" == "$PODMAN_TEST_IMAGE_FQN" ]; then
             found_needed_image=1
         else
-            echo "# setup_standard_environment: podman rmi $1 & $2" >&3
+            echo "# setup(): removing stray images" >&3
-            podman rmi --force "$1" >/dev/null 2>&1 || true
+            run_podman rmi --force "$1" >/dev/null 2>&1 || true
-            podman rmi --force "$2" >/dev/null 2>&1 || true
+            run_podman rmi --force "$2" >/dev/null 2>&1 || true
         fi
     done
 
@@ -43,11 +46,22 @@ function basic_setup() {
     if [ -z "$found_needed_image" ]; then
         run_podman pull "$PODMAN_TEST_IMAGE_FQN"
     fi
+
+    # Argh. Although BATS provides $BATS_TMPDIR, it's just /tmp!
+    # That's bloody worthless. Let's make our own, in which subtests
+    # can write whatever they like and trust that it'll be deleted
+    # on cleanup.
+    # TODO: do this outside of setup, so it carries across tests?
+    PODMAN_TMPDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-/tmp} podman_bats.XXXXXX)
 }
 
-# Basic teardown: remove all containers
+# Basic teardown: remove all pods and containers
 function basic_teardown() {
-    run_podman rm --all --force
+    echo "# [teardown]" >&2
+    run_podman '?' pod rm --all --force
+    run_podman '?'     rm --all --force
+
+    /bin/rm -rf $PODMAN_TMPDIR
 }
 
 
@@ -118,10 +132,12 @@ function run_podman() {
     esac
 
     # stdout is only emitted upon error; this echo is to help a debugger
-    echo "\$ $PODMAN $@"
+    echo "\$ $PODMAN $*"
     run timeout --foreground -v --kill=10 $PODMAN_TIMEOUT $PODMAN "$@"
     # without "quotes", multiple lines are glommed together into one
-    echo "$output"
+    if [ -n "$output" ]; then
+        echo "$output"
+    fi
     if [ "$status" -ne 0 ]; then
         echo -n "[ rc=$status ";
         if [ -n "$expected_rc" ]; then
@@ -143,21 +159,22 @@ function run_podman() {
 
     if [ -n "$expected_rc" ]; then
         if [ "$status" -ne "$expected_rc" ]; then
-            die "FAIL: exit code is $status; expected $expected_rc"
+            die "exit code is $status; expected $expected_rc"
         fi
     fi
 }
 
 
-# Wait for 'READY' in container output
+# Wait for certain output from a container, indicating that it's ready.
-function wait_for_ready {
+function wait_for_output {
-    local cid=
     local sleep_delay=5
-    local how_long=60
+    local how_long=$PODMAN_TIMEOUT
+    local expect=
+    local cid=
 
     # Arg processing. A single-digit number is how long to sleep between
-    # iterations; a 2- or 3-digit number is the total time to wait; anything
+    # iterations; a 2- or 3-digit number is the total time to wait; all
-    # else is the container ID or name to wait on.
+    # else are, in order, the string to expect and the container name/ID.
     local i
     for i in "$@"; do
         if expr "$i" : '[0-9]\+$' >/dev/null; then
@@ -166,6 +183,8 @@ function wait_for_ready {
             else
                 how_long=$i
             fi
+        elif [ -z "$expect" ]; then
+            expect=$i
         else
             cid=$i
         fi
@@ -176,14 +195,19 @@ function wait_for_ready {
     t1=$(expr $SECONDS + $how_long)
     while [ $SECONDS -lt $t1 ]; do
         run_podman logs $cid
-        if expr "$output" : ".*READY" >/dev/null; then
+        if expr "$output" : ".*$expect" >/dev/null; then
             return
         fi
 
         sleep $sleep_delay
     done
 
-    die "FAIL: timed out waiting for READY from $cid"
+    die "timed out waiting for '$expect' from $cid"
+}
+
+# Shortcut for the lazy
+function wait_for_ready {
+    wait_for_output 'READY' "$@"
 }
 
 # END   podman helpers
@@ -206,7 +230,9 @@ function skip_if_rootless() {
 #  die  #  Abort with helpful message
 #########
 function die() {
-    echo "# $*" >&2
+    echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv"  >&2
+    echo "#| FAIL: $*"                                           >&2
+    echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2
     false
 }
 
@@ -232,16 +258,24 @@ function is() {
         if [ -z "$actual" ]; then
             return
         fi
-        die "$testname:\n#  expected no output; got %q\n" "$actual"
+        expect='[no output]'
-    fi
+    elif expr "$actual" : "$expect" >/dev/null; then
-
-    if expr "$actual" : "$expect" >/dev/null; then
         return
     fi
 
-    # This is a multi-line message, so let's format it ourself (not via die)
+    # This is a multi-line message, which may in turn contain multi-line
-    printf "# $testname:\n#  expected: %q\n#    actual: %q\n"    \
+    # output, so let's format it ourself, readably
-           "$expect" "$actual" >&2
+    local -a actual_split
+    readarray -t actual_split <<<"$actual"
+    printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2
+    printf "#|     FAIL: $testname\n"                          >&2
+    printf "#| expected: '%s'\n" "$expect"                     >&2
+    printf "#|   actual: '%s'\n" "${actual_split[0]}"          >&2
+    local line
+    for line in "${actual_split[@]:1}"; do
+        printf "#|         > '%s'\n" "$line"                   >&2
+    done
+    printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2
     false
 }