mirror of
https://github.com/containers/podman.git
synced 2025-10-31 18:08:51 +08:00
dbfa201c9a
Simply because it's been a while since the last testimage
build, and I want to confirm that our image build process
still works.
Added /home/podman/healthcheck. This saves us having to
podman-build on each healthcheck test. Removed now-
unneeded _build_health_check_image helper.
testimage: bump alpine 3.16.2 to 3.19.0
systemd-image: f38 to f39
- tzdata now requires dnf **install**, not reinstall
(this is exactly the sort of thing I was looking for)
PROBLEMS DISCOVERED:
- in e2e, fedoraMinimal is now == SYSTEMD_IMAGE. This
screws up some of the image-count tests (CACHE_IMAGES).
- "alter tarball" system test now barfs with tar < 1.35.
TODO: completely replace fedoraMinimal with SYSTEMD_IMAGE
in all tests.
Signed-off-by: Ed Santiago <santiago@redhat.com>
1183 lines
38 KiB
Bash
1183 lines
38 KiB
Bash
# -*- bash -*-
|
|
|
|
# Podman command to run; may be podman-remote
|
|
PODMAN=${PODMAN:-podman}
|
|
QUADLET=${QUADLET:-/usr/libexec/podman/quadlet}
|
|
|
|
# crun or runc, unlikely to change. Cache, because it's expensive to determine.
|
|
PODMAN_RUNTIME=
|
|
|
|
# Standard image to use for most tests
|
|
PODMAN_TEST_IMAGE_REGISTRY=${PODMAN_TEST_IMAGE_REGISTRY:-"quay.io"}
|
|
PODMAN_TEST_IMAGE_USER=${PODMAN_TEST_IMAGE_USER:-"libpod"}
|
|
PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"testimage"}
|
|
PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"20240123"}
|
|
PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG"
|
|
|
|
# Larger image containing systemd tools.
|
|
PODMAN_SYSTEMD_IMAGE_NAME=${PODMAN_SYSTEMD_IMAGE_NAME:-"systemd-image"}
|
|
PODMAN_SYSTEMD_IMAGE_TAG=${PODMAN_SYSTEMD_IMAGE_TAG:-"20240124"}
|
|
PODMAN_SYSTEMD_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_SYSTEMD_IMAGE_NAME:$PODMAN_SYSTEMD_IMAGE_TAG"
|
|
|
|
# Remote image that we *DO NOT* fetch or keep by default; used for testing pull
|
|
# This has changed in 2021, from 0 through 3, various iterations of getting
|
|
# multiarch to work. It should change only very rarely.
|
|
PODMAN_NONLOCAL_IMAGE_TAG=${PODMAN_NONLOCAL_IMAGE_TAG:-"00000004"}
|
|
PODMAN_NONLOCAL_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_NONLOCAL_IMAGE_TAG"
|
|
|
|
# Because who wants to spell that out each time?
|
|
IMAGE=$PODMAN_TEST_IMAGE_FQN
|
|
SYSTEMD_IMAGE=$PODMAN_SYSTEMD_IMAGE_FQN
|
|
|
|
# Default timeout for a podman command.
|
|
PODMAN_TIMEOUT=${PODMAN_TIMEOUT:-120}
|
|
|
|
# Prompt to display when logging podman commands; distinguish root/rootless
|
|
_LOG_PROMPT='$'
|
|
if [ $(id -u) -eq 0 ]; then
|
|
_LOG_PROMPT='#'
|
|
fi
|
|
|
|
###############################################################################
|
|
# BEGIN tools for fetching & caching test images
|
|
#
|
|
# Registries are flaky: any time we have to pull an image, that's a risk.
|
|
#
|
|
|
|
# Store in a semipermanent location. Not important for CI, but nice for
|
|
# developers so test restarts don't hang fetching images.
|
|
export PODMAN_IMAGECACHE=${BATS_TMPDIR:-/tmp}/podman-systest-imagecache-$(id -u)
|
|
mkdir -p ${PODMAN_IMAGECACHE}
|
|
|
|
function _prefetch() {
|
|
local want=$1
|
|
|
|
# Do we already have it in image store?
|
|
run_podman '?' image exists "$want"
|
|
if [[ $status -eq 0 ]]; then
|
|
return
|
|
fi
|
|
|
|
# No image. Do we have it already cached? (Replace / and : with --)
|
|
local cachename=$(sed -e 's;[/:];--;g' <<<"$want")
|
|
local cachepath="${PODMAN_IMAGECACHE}/${cachename}.tar"
|
|
if [[ ! -e "$cachepath" ]]; then
|
|
# Not cached. Fetch it and cache it. Retry twice, because of flakes.
|
|
cmd="skopeo copy --preserve-digests docker://$want oci-archive:$cachepath"
|
|
echo "$_LOG_PROMPT $cmd"
|
|
run $cmd
|
|
echo "$output"
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "# 'pull $want' failed, will retry..." >&3
|
|
sleep 5
|
|
|
|
run $cmd
|
|
echo "$output"
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "# 'pull $want' failed again, will retry one last time..." >&3
|
|
sleep 30
|
|
$cmd
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Kludge alert.
|
|
# Skopeo has no --storage-driver, --root, or --runroot flags; those
|
|
# need to be expressed in the destination string inside [brackets].
|
|
# See containers-transports(5). So if we see those options in
|
|
# _PODMAN_TEST_OPTS, transmogrify $want into skopeo form.
|
|
skopeo_opts=''
|
|
driver="$(expr "$_PODMAN_TEST_OPTS" : ".*--storage-driver \([^ ]\+\)" || true)"
|
|
if [[ -n "$driver" ]]; then
|
|
skopeo_opts+="$driver@"
|
|
fi
|
|
|
|
altroot="$(expr "$_PODMAN_TEST_OPTS" : ".*--root \([^ ]\+\)" || true)"
|
|
if [[ -n "$altroot" ]] && [[ -d "$altroot" ]]; then
|
|
skopeo_opts+="$altroot"
|
|
|
|
altrunroot="$(expr "$_PODMAN_TEST_OPTS" : ".*--runroot \([^ ]\+\)" || true)"
|
|
if [[ -n "$altrunroot" ]] && [[ -d "$altrunroot" ]]; then
|
|
skopeo_opts+="+$altrunroot"
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "$skopeo_opts" ]]; then
|
|
want="[$skopeo_opts]$want"
|
|
fi
|
|
|
|
# Cached image is now guaranteed to exist. Be sure to load it
|
|
# with skopeo, not podman, in order to preserve metadata
|
|
cmd="skopeo copy --all oci-archive:$cachepath containers-storage:$want"
|
|
echo "$_LOG_PROMPT $cmd"
|
|
$cmd
|
|
}
|
|
|
|
|
|
# Wrapper for skopeo, because skopeo doesn't work rootless if $XDG is unset
|
|
# (as it is in RHEL gating): it defaults to /run/containers/<uid>, which
|
|
# of course is a root-only dir, hence fails with permission denied.
|
|
# -- https://github.com/containers/skopeo/issues/823
|
|
function skopeo() {
|
|
local xdg=${XDG_RUNTIME_DIR}
|
|
if [ -z "$xdg" ]; then
|
|
if is_rootless; then
|
|
xdg=/run/user/$(id -u)
|
|
fi
|
|
fi
|
|
XDG_RUNTIME_DIR=${xdg} command skopeo "$@"
|
|
}
|
|
|
|
# END tools for fetching & caching test images
|
|
###############################################################################
|
|
# BEGIN setup/teardown tools
|
|
|
|
# Provide common setup and teardown functions, but do not name them such!
|
|
# That way individual tests can override with their own setup/teardown,
|
|
# while retaining the ability to include these if they so desire.
|
|
|
|
# Setup helper: establish a test environment with exactly the images needed
|
|
function basic_setup() {
|
|
# Clean up all containers
|
|
run_podman rm -t 0 --all --force --ignore
|
|
|
|
# ...including external (buildah) ones
|
|
run_podman ps --all --external --format '{{.ID}} {{.Names}}'
|
|
for line in "${lines[@]}"; do
|
|
set $line
|
|
echo "# setup(): removing stray external container $1 ($2)" >&3
|
|
run_podman '?' rm -f $1
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "# [setup] $_LOG_PROMPT podman rm -f $1" >&3
|
|
for errline in "${lines[@]}"; do
|
|
echo "# $errline" >&3
|
|
done
|
|
fi
|
|
done
|
|
|
|
# Clean up all images except those desired.
|
|
# 2023-06-26 REMINDER: it is tempting to think that this is clunky,
|
|
# wouldn't it be safer/cleaner to just 'rmi -a' then '_prefetch $IMAGE'?
|
|
# Yes, but it's also tremendously slower: 29m for a CI run, to 39m.
|
|
# Image loads are slow.
|
|
found_needed_image=
|
|
run_podman '?' images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}'
|
|
|
|
for line in "${lines[@]}"; do
|
|
set $line
|
|
if [[ "$1" == "$PODMAN_TEST_IMAGE_FQN" ]]; then
|
|
if [[ -z "$PODMAN_TEST_IMAGE_ID" ]]; then
|
|
# This will probably only trigger the 2nd time through setup
|
|
PODMAN_TEST_IMAGE_ID=$2
|
|
fi
|
|
found_needed_image=1
|
|
elif [[ "$1" == "$PODMAN_SYSTEMD_IMAGE_FQN" ]]; then
|
|
# This is a big image, don't force unnecessary pulls
|
|
:
|
|
else
|
|
# Always remove image that doesn't match by name
|
|
echo "# setup(): removing stray image $1" >&3
|
|
run_podman rmi --force "$1" >/dev/null 2>&1 || true
|
|
|
|
# Tagged image will have same IID as our test image; don't rmi it.
|
|
if [[ $2 != "$PODMAN_TEST_IMAGE_ID" ]]; then
|
|
echo "# setup(): removing stray image $2" >&3
|
|
run_podman rmi --force "$2" >/dev/null 2>&1 || true
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Make sure desired image is present
|
|
if [[ -z "$found_needed_image" ]]; then
|
|
_prefetch $PODMAN_TEST_IMAGE_FQN
|
|
fi
|
|
|
|
# Temporary subdirectory, in which tests can write whatever they like
|
|
# and trust that it'll be deleted on cleanup.
|
|
# (BATS v1.3 and above provide $BATS_TEST_TMPDIR, but we still use
|
|
# ancient BATS (v1.1) in RHEL gating tests.)
|
|
PODMAN_TMPDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-/tmp} podman_bats.XXXXXX)
|
|
|
|
# runtime is not likely to change
|
|
if [[ -z "$PODMAN_RUNTIME" ]]; then
|
|
PODMAN_RUNTIME=$(podman_runtime)
|
|
fi
|
|
|
|
# In the unlikely event that a test runs is() before a run_podman()
|
|
MOST_RECENT_PODMAN_COMMAND=
|
|
|
|
# Test filenames must match ###-name.bats; use "[###] " as prefix
|
|
run expr "$BATS_TEST_FILENAME" : "^.*/\([0-9]\{3\}\)-[^/]\+\.bats\$"
|
|
BATS_TEST_NAME_PREFIX="[${output}] "
|
|
|
|
# By default, assert() and die() cause an immediate test failure.
|
|
# Under special circumstances (usually long test loops), tests
|
|
# can call defer-assertion-failures() to continue going, the
|
|
# idea being that a large number of failures can show patterns.
|
|
ASSERTION_FAILURES=
|
|
immediate-assertion-failures
|
|
}
|
|
|
|
# bail-now is how we terminate a test upon assertion failure.
|
|
# By default, and the vast majority of the time, it just triggers
|
|
# immediate test termination; but see defer-assertion-failures, below.
|
|
function bail-now() {
|
|
# "false" does not apply to "bail now"! It means "nonzero exit",
|
|
# which BATS interprets as "yes, bail immediately".
|
|
false
|
|
}
|
|
|
|
# Invoked on teardown: will terminate immediately if there have been
|
|
# any deferred test failures; otherwise will reset back to immediate
|
|
# test termination on future assertions.
|
|
function immediate-assertion-failures() {
|
|
function bail-now() {
|
|
false
|
|
}
|
|
|
|
# Any backlog?
|
|
if [[ -n "$ASSERTION_FAILURES" ]]; then
|
|
local n=${#ASSERTION_FAILURES}
|
|
ASSERTION_FAILURES=
|
|
die "$n test assertions failed. Search for 'FAIL:' above this line." >&2
|
|
fi
|
|
}
|
|
|
|
# Used in special test circumstances--typically multi-condition loops--to
|
|
# continue going even on assertion failures. The test will fail afterward,
|
|
# usually in teardown. This can be useful to show failure patterns.
|
|
function defer-assertion-failures() {
|
|
function bail-now() {
|
|
ASSERTION_FAILURES+="!"
|
|
}
|
|
}
|
|
|
|
# Basic teardown: remove all pods and containers
|
|
function basic_teardown() {
|
|
echo "# [teardown]" >&2
|
|
local actions=(
|
|
"pod rm -t 0 --all --force --ignore"
|
|
"rm -t 0 --all --force --ignore"
|
|
"network prune --force"
|
|
"volume rm -a -f"
|
|
)
|
|
for action in "${actions[@]}"; do
|
|
run_podman '?' $action
|
|
|
|
# The -f commands should never exit nonzero, but if they do we want
|
|
# to know about it.
|
|
# FIXME: someday: also test for [[ -n "$output" ]] - can't do this
|
|
# yet because too many tests don't clean up their containers
|
|
if [[ $status -ne 0 ]]; then
|
|
echo "# [teardown] $_LOG_PROMPT podman $action" >&3
|
|
for line in "${lines[*]}"; do
|
|
echo "# $line" >&3
|
|
done
|
|
|
|
# Special case for timeout: check for locks (#18514)
|
|
if [[ $status -eq 124 ]]; then
|
|
echo "# [teardown] $_LOG_PROMPT podman system locks" >&3
|
|
run $PODMAN system locks
|
|
for line in "${lines[*]}"; do
|
|
echo "# $line" >&3
|
|
done
|
|
fi
|
|
fi
|
|
done
|
|
|
|
command rm -rf $PODMAN_TMPDIR
|
|
immediate-assertion-failures
|
|
}
|
|
|
|
|
|
# Provide the above as default methods.
|
|
function setup() {
|
|
basic_setup
|
|
}
|
|
|
|
function teardown() {
|
|
basic_teardown
|
|
}
|
|
|
|
|
|
# Helpers useful for tests running rmi
|
|
function archive_image() {
|
|
local image=$1
|
|
|
|
# FIXME: refactor?
|
|
archive_basename=$(echo $1 | tr -c a-zA-Z0-9._- _)
|
|
archive=$BATS_TMPDIR/$archive_basename.tar
|
|
|
|
run_podman save -o $archive $image
|
|
}
|
|
|
|
function restore_image() {
|
|
local image=$1
|
|
|
|
archive_basename=$(echo $1 | tr -c a-zA-Z0-9._- _)
|
|
archive=$BATS_TMPDIR/$archive_basename.tar
|
|
|
|
run_podman restore $archive
|
|
}
|
|
|
|
# END setup/teardown tools
|
|
###############################################################################
|
|
# BEGIN podman helpers
|
|
|
|
# Displays '[HH:MM:SS.NNNNN]' in command output. logformatter relies on this.
|
|
function timestamp() {
|
|
date +'[%T.%N]'
|
|
}
|
|
|
|
################
|
|
# run_podman # Invoke $PODMAN, with timeout, using BATS 'run'
|
|
################
|
|
#
|
|
# This is the preferred mechanism for invoking podman: first, it
|
|
# invokes $PODMAN, which may be 'podman-remote' or '/some/path/podman'.
|
|
#
|
|
# Second, we use 'timeout' to abort (with a diagnostic) if something
|
|
# takes too long; this is preferable to a CI hang.
|
|
#
|
|
# Third, we log the command run and its output. This doesn't normally
|
|
# appear in BATS output, but it will if there's an error.
|
|
#
|
|
# Next, we check exit status. Since the normal desired code is 0,
|
|
# that's the default; but the first argument can override:
|
|
#
|
|
# run_podman 125 nonexistent-subcommand
|
|
# run_podman '?' some-other-command # let our caller check status
|
|
#
|
|
# Since we use the BATS 'run' mechanism, $output and $status will be
|
|
# defined for our caller.
|
|
#
|
|
function run_podman() {
|
|
# Number as first argument = expected exit code; default 0
|
|
# "0+[we]" = require success, but allow warnings/errors
|
|
local expected_rc=0
|
|
local allowed_levels="dit"
|
|
case "$1" in
|
|
0\+[we]*) allowed_levels+=$(expr "$1" : "^0+\([we]\+\)"); shift;;
|
|
[0-9]) expected_rc=$1; shift;;
|
|
[1-9][0-9]) expected_rc=$1; shift;;
|
|
[12][0-9][0-9]) expected_rc=$1; shift;;
|
|
'?') expected_rc= ; shift;; # ignore exit code
|
|
esac
|
|
|
|
# Remember command args, for possible use in later diagnostic messages
|
|
MOST_RECENT_PODMAN_COMMAND="podman $*"
|
|
|
|
# BATS >= 1.5.0 treats 127 as a special case, adding a big nasty warning
|
|
# at the end of the test run if any command exits thus. Silence it.
|
|
# https://bats-core.readthedocs.io/en/stable/warnings/BW01.html
|
|
local silence127=
|
|
if [[ "$expected_rc" = "127" ]]; then
|
|
# We could use "-127", but that would cause BATS to fail if the
|
|
# command exits any other status -- and default BATS failure messages
|
|
# are much less helpful than the run_podman ones. "!" is more flexible.
|
|
silence127="!"
|
|
fi
|
|
|
|
# stdout is only emitted upon error; this printf is to help in debugging
|
|
printf "\n%s %s %s %s\n" "$(timestamp)" "$_LOG_PROMPT" "$PODMAN" "$*"
|
|
# BATS hangs if a subprocess remains and keeps FD 3 open; this happens
|
|
# if podman crashes unexpectedly without cleaning up subprocesses.
|
|
run $silence127 timeout --foreground -v --kill=10 $PODMAN_TIMEOUT $PODMAN $_PODMAN_TEST_OPTS "$@" 3>/dev/null
|
|
# without "quotes", multiple lines are glommed together into one
|
|
if [ -n "$output" ]; then
|
|
echo "$(timestamp) $output"
|
|
|
|
# FIXME FIXME FIXME: instrumenting to track down #15488. Please
|
|
# remove once that's fixed. We include the args because, remember,
|
|
# bats only shows output on error; it's possible that the first
|
|
# instance of the metacopy warning happens in a test that doesn't
|
|
# check output, hence doesn't fail.
|
|
if [[ "$output" =~ Ignoring.global.metacopy.option ]]; then
|
|
echo "# YO! metacopy warning triggered by: podman $*" >&3
|
|
fi
|
|
fi
|
|
if [ "$status" -ne 0 ]; then
|
|
echo -n "$(timestamp) [ rc=$status ";
|
|
if [ -n "$expected_rc" ]; then
|
|
if [ "$status" -eq "$expected_rc" ]; then
|
|
echo -n "(expected) ";
|
|
else
|
|
echo -n "(** EXPECTED $expected_rc **) ";
|
|
fi
|
|
fi
|
|
echo "]"
|
|
fi
|
|
|
|
if [ "$status" -eq 124 ]; then
|
|
if expr "$output" : ".*timeout: sending" >/dev/null; then
|
|
# It's possible for a subtest to _want_ a timeout
|
|
if [[ "$expected_rc" != "124" ]]; then
|
|
echo "*** TIMED OUT ***"
|
|
false
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if [ -n "$expected_rc" ]; then
|
|
if [ "$status" -ne "$expected_rc" ]; then
|
|
die "exit code is $status; expected $expected_rc"
|
|
fi
|
|
fi
|
|
|
|
# Check for "level=<unexpected>" in output, because a successful command
|
|
# should never issue unwanted warnings or errors. The "0+w" convention
|
|
# (see top of function) allows our caller to indicate that warnings are
|
|
# expected, e.g., "podman stop" without -t0.
|
|
if [[ $status -eq 0 ]]; then
|
|
# FIXME: don't do this on Debian or RHEL. runc is way too buggy:
|
|
# - #11784 - lstat /sys/fs/.../*.scope: ENOENT
|
|
# - #11785 - cannot toggle freezer: cgroups not configured
|
|
# As of January 2024 the freezer one seems to be fixed in Debian-runc
|
|
# but not in RHEL8-runc. The lstat one is closed-wontfix.
|
|
if [[ $PODMAN_RUNTIME != "runc" ]]; then
|
|
# FIXME: All kube commands emit unpredictable errors:
|
|
# "Storage for container <X> has been removed"
|
|
# "no container with ID <X> found in database"
|
|
# These are level=error but we still get exit-status 0.
|
|
# Just skip all kube commands completely
|
|
if [[ ! "$*" =~ kube ]]; then
|
|
if [[ "$output" =~ level=[^${allowed_levels}] ]]; then
|
|
die "Command succeeded, but issued unexpected warnings"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
}
|
|
|
|
|
|
# Wait for certain output from a container, indicating that it's ready.
|
|
function wait_for_output {
|
|
local sleep_delay=1
|
|
local how_long=$PODMAN_TIMEOUT
|
|
local expect=
|
|
local cid=
|
|
|
|
# Arg processing. A single-digit number is how long to sleep between
|
|
# iterations; a 2- or 3-digit number is the total time to wait; all
|
|
# else are, in order, the string to expect and the container name/ID.
|
|
local i
|
|
for i in "$@"; do
|
|
if expr "$i" : '[0-9]\+$' >/dev/null; then
|
|
if [ $i -le 9 ]; then
|
|
sleep_delay=$i
|
|
else
|
|
how_long=$i
|
|
fi
|
|
elif [ -z "$expect" ]; then
|
|
expect=$i
|
|
else
|
|
cid=$i
|
|
fi
|
|
done
|
|
|
|
[ -n "$cid" ] || die "FATAL: wait_for_output: no container name/ID in '$*'"
|
|
|
|
t1=$(expr $SECONDS + $how_long)
|
|
while [ $SECONDS -lt $t1 ]; do
|
|
run_podman 0+w logs $cid
|
|
logs=$output
|
|
if expr "$logs" : ".*$expect" >/dev/null; then
|
|
return
|
|
fi
|
|
|
|
# Barf if container is not running
|
|
run_podman inspect --format '{{.State.Running}}' $cid
|
|
if [ $output != "true" ]; then
|
|
run_podman inspect --format '{{.State.ExitCode}}' $cid
|
|
exitcode=$output
|
|
|
|
# One last chance: maybe the container exited just after logs cmd
|
|
run_podman 0+w logs $cid
|
|
if expr "$logs" : ".*$expect" >/dev/null; then
|
|
return
|
|
fi
|
|
|
|
die "Container exited (status: $exitcode) before we saw '$expect': $logs"
|
|
fi
|
|
|
|
sleep $sleep_delay
|
|
done
|
|
|
|
die "timed out waiting for '$expect' from $cid"
|
|
}
|
|
|
|
# Shortcut for the lazy
|
|
function wait_for_ready {
|
|
wait_for_output 'READY' "$@"
|
|
}
|
|
|
|
###################
|
|
# wait_for_file # Returns once file is available on host
|
|
###################
|
|
function wait_for_file() {
|
|
local file=$1 # The path to the file
|
|
local _timeout=${2:-5} # Optional; default 5 seconds
|
|
|
|
# Wait
|
|
while [ $_timeout -gt 0 ]; do
|
|
test -e $file && return
|
|
sleep 1
|
|
_timeout=$(( $_timeout - 1 ))
|
|
done
|
|
|
|
die "Timed out waiting for $file"
|
|
}
|
|
|
|
###########################
|
|
# wait_for_file_content # Like wait_for_output, but with files (not ctrs)
|
|
###########################
|
|
function wait_for_file_content() {
|
|
local file=$1 # The path to the file
|
|
local content=$2 # What to expect in the file
|
|
local _timeout=${3:-5} # Optional; default 5 seconds
|
|
|
|
while :; do
|
|
grep -q "$content" "$file" && return
|
|
|
|
test $_timeout -gt 0 || die "Timed out waiting for '$content' in $file"
|
|
|
|
_timeout=$(( $_timeout - 1 ))
|
|
sleep 1
|
|
|
|
# For debugging. Note that file does not necessarily exist yet.
|
|
if [[ -e "$file" ]]; then
|
|
echo "[ wait_for_file_content: retrying wait for '$content' in: ]"
|
|
sed -e 's/^/[ /' -e 's/$/ ]/' <"$file"
|
|
else
|
|
echo "[ wait_for_file_content: $file does not exist (yet) ]"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# END podman helpers
|
|
###############################################################################
|
|
# BEGIN miscellaneous tools
|
|
|
|
# Shortcuts for common needs:
|
|
function is_rootless() {
|
|
[ "$(id -u)" -ne 0 ]
|
|
}
|
|
|
|
function is_remote() {
|
|
[[ "$PODMAN" =~ -remote ]]
|
|
}
|
|
|
|
function is_cgroupsv1() {
|
|
# WARNING: This will break if there's ever a cgroups v3
|
|
! is_cgroupsv2
|
|
}
|
|
|
|
# True if cgroups v2 are enabled
|
|
function is_cgroupsv2() {
|
|
cgroup_type=$(stat -f -c %T /sys/fs/cgroup)
|
|
test "$cgroup_type" = "cgroup2fs"
|
|
}
|
|
|
|
# True if podman is using netavark
|
|
function is_netavark() {
|
|
run_podman info --format '{{.Host.NetworkBackend}}'
|
|
if [[ "$output" =~ netavark ]]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
function is_aarch64() {
|
|
[ "$(uname -m)" == "aarch64" ]
|
|
}
|
|
|
|
function selinux_enabled() {
|
|
/usr/sbin/selinuxenabled 2> /dev/null
|
|
}
|
|
|
|
# Returns the OCI runtime *basename* (typically crun or runc). Much as we'd
|
|
# love to cache this result, we probably shouldn't.
|
|
function podman_runtime() {
|
|
# This function is intended to be used as '$(podman_runtime)', i.e.
|
|
# our caller wants our output. It's unsafe to use run_podman().
|
|
runtime=$($PODMAN $_PODMAN_TEST_OPTS info --format '{{ .Host.OCIRuntime.Name }}' 2>/dev/null)
|
|
basename "${runtime:-[null]}"
|
|
}
|
|
|
|
# Returns the storage driver: 'overlay' or 'vfs'
|
|
function podman_storage_driver() {
|
|
run_podman info --format '{{.Store.GraphDriverName}}' >/dev/null
|
|
# Should there ever be a new driver
|
|
case "$output" in
|
|
overlay) ;;
|
|
vfs) ;;
|
|
*) die "Unknown storage driver '$output'; if this is a new driver, please review uses of this function in tests." ;;
|
|
esac
|
|
echo "$output"
|
|
}
|
|
|
|
# Given a (scratch) directory path, returns a set of command-line options
|
|
# for running an isolated podman that will not step on system podman. Set:
|
|
# - rootdir, so we don't clobber real images or storage;
|
|
# - tmpdir, so we use an isolated DB; and
|
|
# - runroot, out of an abundance of paranoia
|
|
function podman_isolation_opts() {
|
|
local path=${1?podman_isolation_opts: missing PATH arg}
|
|
|
|
for opt in root runroot tmpdir;do
|
|
mkdir -p $path/$opt
|
|
echo " --$opt $path/$opt"
|
|
done
|
|
}
|
|
|
|
# rhbz#1895105: rootless journald is unavailable except to users in
|
|
# certain magic groups; which our testuser account does not belong to
|
|
# (intentional: that is the RHEL default, so that's the setup we test).
|
|
function journald_unavailable() {
|
|
if ! is_rootless; then
|
|
# root must always have access to journal
|
|
return 1
|
|
fi
|
|
|
|
run journalctl -n 1
|
|
if [[ $status -eq 0 ]]; then
|
|
return 1
|
|
fi
|
|
|
|
if [[ $output =~ permission ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# This should never happen; if it does, it's likely that a subsequent
|
|
# test will fail. This output may help track that down.
|
|
echo "WEIRD: 'journalctl -n 1' failed with a non-permission error:"
|
|
echo "$output"
|
|
return 1
|
|
}
|
|
|
|
# Returns the name of the local pause image.
|
|
function pause_image() {
|
|
# This function is intended to be used as '$(pause_image)', i.e.
|
|
# our caller wants our output. run_podman() messes with output because
|
|
# it emits the command invocation to stdout, hence the redirection.
|
|
run_podman version --format "{{.Server.Version}}-{{.Server.Built}}" >/dev/null
|
|
echo "localhost/podman-pause:$output"
|
|
}
|
|
|
|
# Wait for the pod (1st arg) to transition into the state (2nd arg)
|
|
function _ensure_pod_state() {
|
|
for i in {0..5}; do
|
|
run_podman pod inspect $1 --format "{{.State}}"
|
|
if [[ $output == "$2" ]]; then
|
|
return
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
die "Timed out waiting for pod $1 to enter state $2"
|
|
}
|
|
|
|
# Wait for the container's (1st arg) running state (2nd arg)
|
|
function _ensure_container_running() {
|
|
for i in {0..20}; do
|
|
run_podman container inspect $1 --format "{{.State.Running}}"
|
|
if [[ $output == "$2" ]]; then
|
|
return
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
die "Timed out waiting for container $1 to enter state running=$2"
|
|
}
|
|
|
|
###########################
|
|
# _add_label_if_missing # make sure skip messages include rootless/remote
|
|
###########################
|
|
function _add_label_if_missing() {
|
|
local msg="$1"
|
|
local want="$2"
|
|
|
|
if [ -z "$msg" ]; then
|
|
echo
|
|
elif expr "$msg" : ".*$want" &>/dev/null; then
|
|
echo "$msg"
|
|
else
|
|
echo "[$want] $msg"
|
|
fi
|
|
}
|
|
|
|
######################
|
|
# skip_if_no_ssh # ...with an optional message
|
|
######################
|
|
function skip_if_no_ssh() {
|
|
if no_ssh; then
|
|
local msg=$(_add_label_if_missing "$1" "ssh")
|
|
skip "${msg:-not applicable with no ssh binary}"
|
|
fi
|
|
}
|
|
|
|
######################
|
|
# skip_if_rootless # ...with an optional message
|
|
######################
|
|
function skip_if_rootless() {
|
|
if is_rootless; then
|
|
local msg=$(_add_label_if_missing "$1" "rootless")
|
|
skip "${msg:-not applicable under rootless podman}"
|
|
fi
|
|
}
|
|
|
|
######################
|
|
# skip_if_not_rootless # ...with an optional message
|
|
######################
|
|
function skip_if_not_rootless() {
|
|
if ! is_rootless; then
|
|
local msg=$(_add_label_if_missing "$1" "rootful")
|
|
skip "${msg:-not applicable under rootlfull podman}"
|
|
fi
|
|
}
|
|
|
|
####################
|
|
# skip_if_remote # ...with an optional message
|
|
####################
|
|
function skip_if_remote() {
|
|
if is_remote; then
|
|
local msg=$(_add_label_if_missing "$1" "remote")
|
|
skip "${msg:-test does not work with podman-remote}"
|
|
fi
|
|
}
|
|
|
|
########################
|
|
# skip_if_no_selinux #
|
|
########################
|
|
function skip_if_no_selinux() {
|
|
if [ ! -e /usr/sbin/selinuxenabled ]; then
|
|
skip "selinux not available"
|
|
elif ! /usr/sbin/selinuxenabled; then
|
|
skip "selinux disabled"
|
|
fi
|
|
}
|
|
|
|
#######################
|
|
# skip_if_cgroupsv1 # ...with an optional message
|
|
#######################
|
|
function skip_if_cgroupsv1() {
|
|
if ! is_cgroupsv2; then
|
|
skip "${1:-test requires cgroupsv2}"
|
|
fi
|
|
}
|
|
|
|
#######################
|
|
# skip_if_cgroupsv2 # ...with an optional message
|
|
#######################
|
|
function skip_if_cgroupsv2() {
|
|
if is_cgroupsv2; then
|
|
skip "${1:-test requires cgroupsv1}"
|
|
fi
|
|
}
|
|
|
|
######################
|
|
# skip_if_rootless_cgroupsv1 # ...with an optional message
|
|
######################
|
|
function skip_if_rootless_cgroupsv1() {
|
|
if is_rootless; then
|
|
if ! is_cgroupsv2; then
|
|
local msg=$(_add_label_if_missing "$1" "rootless cgroupvs1")
|
|
skip "${msg:-not supported as rootless under cgroupsv1}"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
##################################
|
|
# skip_if_journald_unavailable # rhbz#1895105: rootless journald permissions
|
|
##################################
|
|
function skip_if_journald_unavailable {
|
|
if journald_unavailable; then
|
|
skip "Cannot use rootless journald on this system"
|
|
fi
|
|
}
|
|
|
|
function skip_if_aarch64 {
|
|
if is_aarch64; then
|
|
skip "${msg:-Cannot run this test on aarch64 systems}"
|
|
fi
|
|
}
|
|
|
|
#########
|
|
# die # Abort with helpful message
|
|
#########
|
|
function die() {
|
|
# FIXME: handle multi-line output
|
|
echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" >&2
|
|
echo "#| FAIL: $*" >&2
|
|
echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2
|
|
bail-now
|
|
}
|
|
|
|
############
|
|
# assert # Compare actual vs expected string; fail if mismatch
|
|
############
|
|
#
|
|
# Compares string (default: $output) against the given string argument.
|
|
# By default we do an exact-match comparison against $output, but there
|
|
# are two different ways to invoke us, each with an optional description:
|
|
#
|
|
# assert "EXPECT" [DESCRIPTION]
|
|
# assert "RESULT" "OP" "EXPECT" [DESCRIPTION]
|
|
#
|
|
# The first form (one or two arguments) does an exact-match comparison
|
|
# of "$output" against "EXPECT". The second (three or four args) compares
|
|
# the first parameter against EXPECT, using the given OPerator. If present,
|
|
# DESCRIPTION will be displayed on test failure.
|
|
#
|
|
# Examples:
|
|
#
|
|
# assert "this is exactly what we expect"
|
|
# assert "${lines[0]}" =~ "^abc" "first line begins with abc"
|
|
#
|
|
function assert() {
|
|
local actual_string="$output"
|
|
local operator='=='
|
|
local expect_string="$1"
|
|
local testname="$2"
|
|
|
|
case "${#*}" in
|
|
0) die "Internal error: 'assert' requires one or more arguments" ;;
|
|
1|2) ;;
|
|
3|4) actual_string="$1"
|
|
operator="$2"
|
|
expect_string="$3"
|
|
testname="$4"
|
|
;;
|
|
*) die "Internal error: too many arguments to 'assert'" ;;
|
|
esac
|
|
|
|
# Comparisons.
|
|
# Special case: there is no !~ operator, so fake it via '! x =~ y'
|
|
local not=
|
|
local actual_op="$operator"
|
|
if [[ $operator == '!~' ]]; then
|
|
not='!'
|
|
actual_op='=~'
|
|
fi
|
|
if [[ $operator == '=' || $operator == '==' ]]; then
|
|
# Special case: we can't use '=' or '==' inside [[ ... ]] because
|
|
# the right-hand side is treated as a pattern... and '[xy]' will
|
|
# not compare literally. There seems to be no way to turn that off.
|
|
if [ "$actual_string" = "$expect_string" ]; then
|
|
return
|
|
fi
|
|
elif [[ $operator == '!=' ]]; then
|
|
# Same special case as above
|
|
if [ "$actual_string" != "$expect_string" ]; then
|
|
return
|
|
fi
|
|
else
|
|
if eval "[[ $not \$actual_string $actual_op \$expect_string ]]"; then
|
|
return
|
|
elif [ $? -gt 1 ]; then
|
|
die "Internal error: could not process 'actual' $operator 'expect'"
|
|
fi
|
|
fi
|
|
|
|
# Test has failed. Get a descriptive test name.
|
|
if [ -z "$testname" ]; then
|
|
testname="${MOST_RECENT_PODMAN_COMMAND:-[no test name given]}"
|
|
fi
|
|
|
|
# Display optimization: the typical case for 'expect' is an
|
|
# exact match ('='), but there are also '=~' or '!~' or '-ge'
|
|
# and the like. Omit the '=' but show the others; and always
|
|
# align subsequent output lines for ease of comparison.
|
|
local op=''
|
|
local ws=''
|
|
if [ "$operator" != '==' ]; then
|
|
op="$operator "
|
|
ws=$(printf "%*s" ${#op} "")
|
|
fi
|
|
|
|
# This is a multi-line message, which may in turn contain multi-line
|
|
# output, so let's format it ourself to make it more readable.
|
|
local expect_split
|
|
mapfile -t expect_split <<<"$expect_string"
|
|
local actual_split
|
|
mapfile -t actual_split <<<"$actual_string"
|
|
|
|
# bash %q is really nice, except for the way it backslashes spaces
|
|
local -a expect_split_q
|
|
for line in "${expect_split[@]}"; do
|
|
local q=$(printf "%q" "$line" | sed -e 's/\\ / /g')
|
|
expect_split_q+=("$q")
|
|
done
|
|
local -a actual_split_q
|
|
for line in "${actual_split[@]}"; do
|
|
local q=$(printf "%q" "$line" | sed -e 's/\\ / /g')
|
|
actual_split_q+=("$q")
|
|
done
|
|
|
|
printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2
|
|
printf "#| FAIL: %s\n" "$testname" >&2
|
|
printf "#| expected: %s%s\n" "$op" "${expect_split_q[0]}" >&2
|
|
local line
|
|
for line in "${expect_split_q[@]:1}"; do
|
|
printf "#| > %s%s\n" "$ws" "$line" >&2
|
|
done
|
|
printf "#| actual: %s%s\n" "$ws" "${actual_split_q[0]}" >&2
|
|
for line in "${actual_split_q[@]:1}"; do
|
|
printf "#| > %s%s\n" "$ws" "$line" >&2
|
|
done
|
|
printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2
|
|
bail-now
|
|
}
|
|
|
|
########
|
|
# is # **DEPRECATED**; see assert() above
|
|
########
|
|
function is() {
|
|
local actual="$1"
|
|
local expect="$2"
|
|
local testname="${3:-${MOST_RECENT_PODMAN_COMMAND:-[no test name given]}}"
|
|
|
|
local is_expr=
|
|
if [ -z "$expect" ]; then
|
|
if [ -z "$actual" ]; then
|
|
# Both strings are empty.
|
|
return
|
|
fi
|
|
expect='[no output]'
|
|
elif [[ "$actual" = "$expect" ]]; then
|
|
# Strings are identical.
|
|
return
|
|
else
|
|
# Strings are not identical. Are there wild cards in our expect string?
|
|
if expr "$expect" : ".*[^\\][\*\[]" >/dev/null; then
|
|
# There is a '[' or '*' without a preceding backslash.
|
|
is_expr=' (using expr)'
|
|
elif [[ "${expect:0:1}" = '[' ]]; then
|
|
# String starts with '[', e.g. checking seconds like '[345]'
|
|
is_expr=' (using expr)'
|
|
fi
|
|
if [[ -n "$is_expr" ]]; then
|
|
if expr "$actual" : "$expect" >/dev/null; then
|
|
return
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# This is a multi-line message, which may in turn contain multi-line
|
|
# output, so let's format it ourself to make it more readable.
|
|
local -a actual_split
|
|
readarray -t actual_split <<<"$actual"
|
|
printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2
|
|
printf "#| FAIL: $testname\n" >&2
|
|
printf "#| expected: '%s'%s\n" "$expect" "$is_expr" >&2
|
|
printf "#| actual: '%s'\n" "${actual_split[0]}" >&2
|
|
local line
|
|
for line in "${actual_split[@]:1}"; do
|
|
printf "#| > '%s'\n" "$line" >&2
|
|
done
|
|
printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2
|
|
bail-now
|
|
}
|
|
|
|
####################
|
|
# allow_warnings # check cmd output for warning messages other than these
|
|
####################
|
|
#
|
|
# HEADS UP: Operates on '$lines' array, so, must be invoked after run_podman
|
|
#
|
|
function allow_warnings() {
|
|
for line in "${lines[@]}"; do
|
|
if [[ "$line" =~ level=[we] ]]; then
|
|
local ok=
|
|
for pattern in "$@"; do
|
|
if [[ "$line" =~ $pattern ]]; then
|
|
ok=ok
|
|
fi
|
|
done
|
|
if [[ -z "$ok" ]]; then
|
|
die "Unexpected warning/error in command results: $line"
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
#####################
|
|
# require_warning # Require the given message, but disallow any others
|
|
#####################
|
|
# Optional 2nd argument is a message to display if warning is missing
|
|
function require_warning() {
|
|
local expect="$1"
|
|
local msg="${2:-Did not find expected warning/error message}"
|
|
assert "$output" =~ "$expect" "$msg"
|
|
allow_warnings "$expect"
|
|
}
|
|
|
|
############
|
|
# dprint # conditional debug message
|
|
############
|
|
#
|
|
# Set PODMAN_TEST_DEBUG to the name of one or more functions you want to debug
|
|
#
|
|
# Examples:
|
|
#
|
|
# $ PODMAN_TEST_DEBUG=parse_table bats .
|
|
# $ PODMAN_TEST_DEBUG="test_podman_images test_podman_run" bats .
|
|
#
|
|
function dprint() {
|
|
test -z "$PODMAN_TEST_DEBUG" && return
|
|
|
|
caller="${FUNCNAME[1]}"
|
|
|
|
# PODMAN_TEST_DEBUG is a space-separated list of desired functions
|
|
# e.g. "parse_table test_podman_images" (or even just "table")
|
|
for want in $PODMAN_TEST_DEBUG; do
|
|
# Check if our calling function matches any of the desired strings
|
|
if expr "$caller" : ".*$want" >/dev/null; then
|
|
echo "# ${FUNCNAME[1]}() : $*" >&3
|
|
return
|
|
fi
|
|
done
|
|
}
|
|
|
|
|
|
#################
|
|
# parse_table # Split a table on '|' delimiters; return space-separated
|
|
#################
|
|
#
|
|
# See sample .bats scripts for examples. The idea is to list a set of
|
|
# tests in a table, then use simple logic to iterate over each test.
|
|
# Columns are separated using '|' (pipe character) because sometimes
|
|
# we need spaces in our fields.
|
|
#
|
|
function parse_table() {
|
|
while read line; do
|
|
test -z "$line" && continue
|
|
|
|
declare -a row=()
|
|
while read col; do
|
|
dprint "col=<<$col>>"
|
|
row+=("$col")
|
|
done < <(echo "$line" | sed -E -e 's/(^|\s)\|(\s|$)/\n /g' | sed -e 's/^ *//' -e 's/\\/\\\\/g')
|
|
# the above seds:
|
|
# 1) Convert '|' to newline, but only if bracketed by spaces or
|
|
# at beginning/end of line (this allows 'foo|bar' in tests);
|
|
# 2) then remove leading whitespace;
|
|
# 3) then double-escape all backslashes
|
|
|
|
printf "%q " "${row[@]}"
|
|
printf "\n"
|
|
done <<<"$1"
|
|
}
|
|
|
|
|
|
###################
|
|
# random_string # Returns a pseudorandom human-readable string
|
|
###################
|
|
#
|
|
# Numeric argument, if present, is desired length of string
|
|
#
|
|
function random_string() {
|
|
local length=${1:-10}
|
|
|
|
head /dev/urandom | tr -dc a-zA-Z0-9 | head -c$length
|
|
}
|
|
|
|
#########################
|
|
# find_exec_pid_files # Returns nothing or exec_pid hash files
|
|
#########################
|
|
#
|
|
# Return exec_pid hash files if exists, otherwise, return nothing
|
|
#
|
|
function find_exec_pid_files() {
|
|
run_podman info --format '{{.Store.RunRoot}}'
|
|
local storage_path="$output"
|
|
if [ -d $storage_path ]; then
|
|
find $storage_path -type f -iname 'exec_pid_*'
|
|
fi
|
|
}
|
|
|
|
|
|
#############################
|
|
# remove_same_dev_warning # Filter out useless warning from output
|
|
#############################
|
|
#
|
|
# On some CI systems, 'podman run --privileged' emits a useless warning:
|
|
#
|
|
# WARNING: The same type, major and minor should not be used for multiple devices.
|
|
#
|
|
# This obviously screws us up when we look at output results.
|
|
#
|
|
# This function removes the warning from $output and $lines. We don't
|
|
# do a full string match because there's another variant of that message:
|
|
#
|
|
# WARNING: Creating device "/dev/null" with same type, major and minor as existing "/dev/foodevdir/null".
|
|
#
|
|
# (We should never again see that precise error ever again, but we could
|
|
# see variants of it).
|
|
#
|
|
function remove_same_dev_warning() {
|
|
# No input arguments. We operate in-place on $output and $lines
|
|
|
|
local i=0
|
|
local -a new_lines=()
|
|
while [[ $i -lt ${#lines[@]} ]]; do
|
|
if expr "${lines[$i]}" : 'WARNING: .* same type, major' >/dev/null; then
|
|
:
|
|
else
|
|
new_lines+=("${lines[$i]}")
|
|
fi
|
|
i=$(( i + 1 ))
|
|
done
|
|
|
|
lines=("${new_lines[@]}")
|
|
output=$(printf '%s\n' "${lines[@]}")
|
|
}
|
|
|
|
# run 'podman help', parse the output looking for 'Available Commands';
|
|
# return that list.
|
|
function _podman_commands() {
|
|
dprint "$@"
|
|
# &>/dev/null prevents duplicate output
|
|
run_podman help "$@" &>/dev/null
|
|
awk '/^Available Commands:/{ok=1;next}/^Options:/{ok=0}ok { print $1 }' <<<"$output" | grep .
|
|
}
|
|
|
|
##########################
|
|
# sleep_to_next_second # Sleep until second rolls over
|
|
##########################
|
|
|
|
function sleep_to_next_second() {
|
|
sleep 0.$(printf '%04d' $((10000 - 10#$(date +%4N))))
|
|
}
|
|
|
|
function wait_for_command_output() {
|
|
local cmd="$1"
|
|
local want="$2"
|
|
local tries=20
|
|
local sleep_delay=0.5
|
|
|
|
case "${#*}" in
|
|
2) ;;
|
|
4) tries="$3"
|
|
sleep_delay="$4"
|
|
;;
|
|
*) die "Internal error: 'wait_for_command_output' requires two or four arguments" ;;
|
|
esac
|
|
|
|
while [[ $tries -gt 0 ]]; do
|
|
echo "$_LOG_PROMPT $cmd"
|
|
run $cmd
|
|
echo "$output"
|
|
if [[ "$output" = "$want" ]]; then
|
|
return
|
|
fi
|
|
|
|
sleep $sleep_delay
|
|
tries=$((tries - 1))
|
|
done
|
|
die "Timed out waiting for '$cmd' to return '$want'"
|
|
}
|
|
|
|
# END miscellaneous tools
|
|
###############################################################################
|