System tests: fixes for RHEL8 gating failures

- tmpfs + noswap test: requires noswap feature in kernel.
  Check for it, and skip if unimplemented. (Root only.
  Rootless test works regardless of kernel).

- podman generate systemd tests: always use --files option,
  because otherwise the "DEPRECATED" warning gets written
  to the systemd unit file.

- kube play tests: yikes. Fix longstanding bugs when checking
  for containers running. This revealed a longstanding bug
  in one test: multi-pod YAML never actually worked. Fixed now.

- run_podman(): that new check-for-warnings code we added
  in #19878, duh, I skipped it on Debian but should've skipped
  when *runc*. Do so now and update the comment. Requires
  minor surgery to podman_runtime() helper to avoid
  infinite recursion.

Signed-off-by: Ed Santiago <santiago@redhat.com>

test/system/060-mount.bats

@@ -308,6 +308,21 @@ EOF
 }
 
 @test "podman mount noswap memory mounts" {
+    # tmpfs+noswap new in kernel 6.x, mid-2023; likely not in RHEL for a while
+    if ! is_rootless; then
+        testmount=$PODMAN_TMPDIR/testmount
+        mkdir $testmount
+        run mount -t tmpfs -o noswap none $testmount
+        if [[ $status -ne 0 ]]; then
+            if [[ $output =~ "bad option" ]]; then
+                skip "requires kernel with tmpfs + noswap support"
+            fi
+            die "Could not test for tmpfs + noswap support: $output"
+        else
+            umount $testmount
+        fi
+    fi
+
     # if volumes source and dest match then pass
     run_podman run --rm --mount type=ramfs,destination=${PODMAN_TMPDIR} $IMAGE stat -f -c "%T" ${PODMAN_TMPDIR}
     is "$output" "ramfs" "ramfs mounted"

test/system/250-systemd.bats

@@ -9,7 +9,7 @@ load helpers.systemd
 SERVICE_NAME="podman_test_$(random_string)"
 
 UNIT_FILE="$UNIT_DIR/$SERVICE_NAME.service"
-TEMPLATE_FILE_PREFIX="$UNIT_DIR/$SERVICE_NAME"
+TEMPLATE_FILE="$UNIT_DIR/$SERVICE_NAME@.service"
 
 function setup() {
     skip_if_remote "systemd tests are meaningless over remote"
@@ -33,11 +33,19 @@ function teardown() {
 
 # Helper to start a systemd service running a container
 function service_setup() {
-    run_podman generate systemd \
+    # January 2024: we can no longer do "run_podman generate systemd" followed
+    # by "echo $output >file", because generate-systemd is deprecated and now
+    # says so loudly, to stderr, with no way to silence it. Since BATS gloms
+    # stdout + stderr, that warning goes to the unit file. (Today's systemd
+    # is forgiving about that, but RHEL8 systemd chokes with EINVAL)
+    (
+        cd $UNIT_DIR
+        run_podman generate systemd --files --name \
                -e http_proxy -e https_proxy -e no_proxy \
                -e HTTP_PROXY -e HTTPS_PROXY -e NO_PROXY \
                --new $cname
-    echo "$output" > "$UNIT_FILE"
+        mv "container-$cname.service" $UNIT_FILE
+    )
     run_podman rm $cname
 
     systemctl daemon-reload
@@ -222,8 +230,12 @@ LISTEN_FDNAMES=listen_fdnames" | sort)
     cname=$(random_string)
     run_podman create --name $cname $IMAGE top
 
-    run_podman generate systemd --template -n $cname
-    echo "$output" > "$TEMPLATE_FILE_PREFIX@.service"
+    # See note in service_setup() above re: using --files
+    (
+        cd $UNIT_DIR
+        run_podman generate systemd --template --files -n $cname
+        mv "container-$cname.service" $TEMPLATE_FILE
+    )
     run_podman rm -f $cname
 
     systemctl daemon-reload
@@ -237,7 +249,7 @@ LISTEN_FDNAMES=listen_fdnames" | sort)
     run systemctl stop "$INSTANCE"
     assert $status -eq 0 "Error stopping systemd unit $INSTANCE: $output"
 
-    rm -f "$TEMPLATE_FILE_PREFIX@.service"
+    rm -f $TEMPLATE_FILE
     systemctl daemon-reload
 }
 

test/system/260-sdnotify.bats

@@ -352,16 +352,18 @@ EOF
         $PODMAN play kube --service-container=true --log-driver journald $yaml_source &>/dev/null &
 
     # Wait for both containers to be running
+    containers_running=
     for i in $(seq 1 20); do
         run_podman "?" container wait $container_a $container_b --condition="running"
         if [[ $status == 0 ]]; then
+            containers_running=1
             break
         fi
         sleep 0.5
         # Just for debugging
         run_podman ps -a
     done
-    if [[ $status != 0 ]]; then
+    if [[ -z "$containers_running" ]]; then
         die "container $container_a and/or $container_b did not start"
     fi
 

test/system/700-play.bats

@@ -185,16 +185,18 @@ EOF
 
     # Wait for the container to be running
     container_a=test_pod-test
+    container_running=
     for i in $(seq 1 20); do
         run_podman "?" container wait $container_a --condition="running"
         if [[ $status == 0 ]]; then
+            container_running=1
             break
         fi
         sleep 0.5
         # Just for debugging
         run_podman ps -a
     done
-    if [[ $status != 0 ]]; then
+    if [[ -z "$container_running" ]]; then
         die "container $container_a did not start"
     fi
 
@@ -581,7 +583,7 @@ EOF
     # Create the YAMl file, with two pods, each with one container
     yaml_source="$PODMAN_TMPDIR/test.yaml"
     for n in 1 2;do
-        _write_test_yaml labels="app: pod$n" name="pod$n" ctrname="ctr$n"
+        _write_test_yaml labels="app: pod$n" name="pod$n" ctrname="ctr$n" command=top
 
         # Separator between two yaml halves
         if [[ $n = 1 ]]; then
@@ -600,17 +602,19 @@ EOF
     service_container="${yaml_sha:0:12}-service"
     # Wait for the containers to be running
     container_1=pod1-ctr1
-    container_2=pod1-ctr2
+    container_2=pod2-ctr2
+    containers_running=
     for i in $(seq 1 20); do
         run_podman "?" container wait $container_1 $container_2 $service_container --condition="running"
         if [[ $status == 0 ]]; then
+            containers_running=1
             break
         fi
         sleep 0.5
         # Just for debugging
         run_podman ps -a
     done
-    if [[ $status != 0 ]]; then
+    if [[ -z "$containers_running" ]]; then
         die "container $container_1, $container_2 and/or $service_container did not start"
     fi
 

test/system/helpers.bash

@@ -4,6 +4,9 @@
 PODMAN=${PODMAN:-podman}
 QUADLET=${QUADLET:-/usr/libexec/podman/quadlet}
 
+# crun or runc, unlikely to change. Cache, because it's expensive to determine.
+PODMAN_RUNTIME=
+
 # Standard image to use for most tests
 PODMAN_TEST_IMAGE_REGISTRY=${PODMAN_TEST_IMAGE_REGISTRY:-"quay.io"}
 PODMAN_TEST_IMAGE_USER=${PODMAN_TEST_IMAGE_USER:-"libpod"}
@@ -195,6 +198,11 @@ function basic_setup() {
     # ancient BATS (v1.1) in RHEL gating tests.)
     PODMAN_TMPDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-/tmp} podman_bats.XXXXXX)
 
+    # runtime is not likely to change
+    if [[ -z "$PODMAN_RUNTIME" ]]; then
+        PODMAN_RUNTIME=$(podman_runtime)
+    fi
+
     # In the unlikely event that a test runs is() before a run_podman()
     MOST_RECENT_PODMAN_COMMAND=
 
@@ -421,10 +429,12 @@ function run_podman() {
     # (see top of function) allows our caller to indicate that warnings are
     # expected, e.g., "podman stop" without -t0.
     if [[ $status -eq 0 ]]; then
-        # FIXME: don't do this on Debian: runc is way, way too flaky:
-        # FIXME: #11784 - lstat /sys/fs/.../*.scope: ENOENT
-        # FIXME: #11785 - cannot toggle freezer: cgroups not configured
-        if [[ ! "${DISTRO_NV}" =~ debian ]]; then
+        # FIXME: don't do this on Debian or RHEL. runc is way too buggy:
+        #   - #11784 - lstat /sys/fs/.../*.scope: ENOENT
+        #   - #11785 - cannot toggle freezer: cgroups not configured
+        # As of January 2024 the freezer one seems to be fixed in Debian-runc
+        # but not in RHEL8-runc. The lstat one is closed-wontfix.
+        if [[ $PODMAN_RUNTIME != "runc" ]]; then
             # FIXME: All kube commands emit unpredictable errors:
             #    "Storage for container <X> has been removed"
             #    "no container with ID <X> found in database"
@@ -589,10 +599,9 @@ function selinux_enabled() {
 # love to cache this result, we probably shouldn't.
 function podman_runtime() {
     # This function is intended to be used as '$(podman_runtime)', i.e.
-    # our caller wants our output. run_podman() messes with output because
-    # it emits the command invocation to stdout, hence the redirection.
-    run_podman info --format '{{ .Host.OCIRuntime.Name }}' >/dev/null
-    basename "${output:-[null]}"
+    # our caller wants our output. It's unsafe to use run_podman().
+    runtime=$($PODMAN $_PODMAN_TEST_OPTS info --format '{{ .Host.OCIRuntime.Name }}' 2>/dev/null)
+    basename "${runtime:-[null]}"
 }
 
 # Returns the storage driver: 'overlay' or 'vfs'