system tests: use CONTAINERS_CONF_OVERRIDE

...not CONTAINERS_CONF. At least for most tests.

Nearly every system test currently using CONTAINERS_CONF=tmpfile
should be using CONTAINERS_CONF_OVERRIDE.

Simple reason: runtime (crun/runc), database_backend (bolt/sqlite),
logger, and other important settings from /etc/c.conf are not
usually written into the tmpfile. Those tests, therefore, are
not running podman as configured on the system.

Much more discussion: #15413

This PR is a prerequisite for enabling sqlite system tests. For
the sake of simplicity and sanity, I choose to submit the sqlite
switch as a separate PR once this passes and merges.

Signed-off-by: Ed Santiago <santiago@redhat.com>

test/system/030-run.bats

@@ -852,11 +852,11 @@ EOF
 [containers]
 oom_score_adj=$oomscore
 EOF
-    CONTAINERS_CONF=$PODMAN_TMPDIR/containers.conf run_podman run --rm $IMAGE cat /proc/self/oom_score_adj
+    CONTAINERS_CONF_OVERRIDE=$PODMAN_TMPDIR/containers.conf run_podman run --rm $IMAGE cat /proc/self/oom_score_adj
     is "$output" "$oomscore" "two more then default oomscore"
 
     oomscore=$((oomscore+1))
-    CONTAINERS_CONF=$PODMAN_TMPDIR/containers.conf run_podman run --oom-score-adj=$oomscore --rm $IMAGE cat /proc/self/oom_score_adj
+    CONTAINERS_CONF_OVERRIDE=$PODMAN_TMPDIR/containers.conf run_podman run --oom-score-adj=$oomscore --rm $IMAGE cat /proc/self/oom_score_adj
     is "$output" "$oomscore" "--oom-score-adj should override containers.conf"
 }
 
@@ -1018,10 +1018,10 @@ $IMAGE--c_ok" \
 read_only=true
 EOF
 
-    CONTAINERS_CONF="$containersconf" run_podman 1 run --rm $IMAGE touch /testro
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman 1 run --rm $IMAGE touch /testro
-    CONTAINERS_CONF="$containersconf" run_podman run --rm --read-only=false $IMAGE touch /testrw
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman run --rm --read-only=false $IMAGE touch /testrw
-    CONTAINERS_CONF="$containersconf" run_podman run --rm $IMAGE touch /tmp/testrw
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman run --rm $IMAGE touch /tmp/testrw
-    CONTAINERS_CONF="$containersconf" run_podman 1 run --rm --read-only-tmpfs=false $IMAGE touch /tmp/testro
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman 1 run --rm --read-only-tmpfs=false $IMAGE touch /tmp/testro
 }
 
 @test "podman run ulimit from containers.conf" {
@@ -1037,9 +1037,9 @@ default_ulimits = [
 ]
 EOF
 
-    CONTAINERS_CONF="$containersconf" run_podman run --rm $IMAGE grep "Max open files" /proc/self/limits
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman run --rm $IMAGE grep "Max open files" /proc/self/limits
     assert "$output" =~ " ${nofile1}  * ${nofile1}  * files"
-    CONTAINERS_CONF="$containersconf" run_podman run --ulimit nofile=${nofile2}:${nofile2} --rm $IMAGE grep "Max open files" /proc/self/limits
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman run --ulimit nofile=${nofile2}:${nofile2} --rm $IMAGE grep "Max open files" /proc/self/limits
     assert "$output" =~ " ${nofile2}  * ${nofile2}  * files"
 }
 

test/system/070-build.bats

@@ -153,7 +153,12 @@ runtime="idonotexist"
 EOF
 
     CONTAINERS_CONF="$containersconf" run_podman 125 build -t build_test $tmpdir
-    is "$output" ".*\"idonotexist\" not found.*" "failed when passing invalid OCI runtime via containers.conf"
+    is "$output" ".*\"idonotexist\" not found.*" \
+       "failed when passing invalid OCI runtime via \$CONTAINERS_CONF"
+
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman 125 build -t build_test $tmpdir
+    is "$output" ".*\"idonotexist\" not found.*" \
+       "failed when passing invalid OCI runtime via \$CONTAINERS_CONF_OVERRIDE"
 }
 
 # Regression from v1.5.0. This test passes fine in v1.5.0, fails in 1.6

test/system/090-events.bats

@@ -149,7 +149,7 @@ function _events_disjunctive_filters() {
 [engine]
 events_logfile_path="$events_file"
 EOF
-    CONTAINERS_CONF="$containersconf" run_podman --events-backend=file pull $IMAGE
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman --events-backend=file pull $IMAGE
     assert "$(< $events_file)" =~ "\"Name\":\"$IMAGE\"" "Image found in events"
 }
 
@@ -163,7 +163,7 @@ function _populate_events_file() {
 }
 
 @test "events log-file rotation" {
-    skip_if_remote "setting CONTAINERS_CONF logger options does not affect remote client"
+    skip_if_remote "setting CONTAINERS_CONF_OVERRIDE logger options does not affect remote client"
 
     # Make sure that the events log file is (not) rotated depending on the
     # settings in containers.conf.
@@ -178,14 +178,14 @@ events_logfile_path="$eventsFile"
 EOF
 
     # Check that a non existing event file does not cause a hang (#15688)
-    CONTAINERS_CONF=$containersConf run_podman events --stream=false
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman events --stream=false
 
     _populate_events_file $eventsFile
 
     # Create events *without* a limit and make sure that it has not been
     # rotated/truncated.
     contentBefore=$(head -n100 $eventsFile)
-    CONTAINERS_CONF=$containersConf run_podman run --rm $IMAGE true
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman run --rm $IMAGE true
     contentAfter=$(head -n100 $eventsFile)
     is "$contentBefore" "$contentAfter" "events file has not been rotated"
 
@@ -210,7 +210,7 @@ EOF
 
     run_podman create $IMAGE
     ctrID=$output
-    CONTAINERS_CONF=$containersConf run_podman rm $ctrID
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman rm $ctrID
     tail -n52 $eventsFile >> $expectedContentAfterTruncation
 
     # Make sure the events file looks as expected.
@@ -218,7 +218,7 @@ EOF
 
     # Make sure that `podman events` can read the file, and that it returns the
     # same amount of events.  We checked the contents before.
-    CONTAINERS_CONF=$containersConf run_podman events --stream=false --since="2022-03-06T11:26:42.723667984+02:00" --format=json
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman events --stream=false --since="2022-03-06T11:26:42.723667984+02:00" --format=json
     assert "${#lines[@]}" = 52 "Number of events returned"
     is "${lines[0]}" "{\"Name\":\"$eventsFile\",\"Status\":\"log-rotation\",\"Time\":\".*\",\"Type\":\"system\",\"Attributes\":{\"io.podman.event.rotate\":\"begin\"}}"
     is "${lines[-2]}" "{\"Name\":\"$eventsFile\",\"Status\":\"log-rotation\",\"Time\":\".*\",\"Type\":\"system\",\"Attributes\":{\"io.podman.event.rotate\":\"end\"}}"
@@ -226,7 +226,7 @@ EOF
 }
 
 @test "events log-file no duplicates" {
-    skip_if_remote "setting CONTAINERS_CONF logger options does not affect remote client"
+    skip_if_remote "setting CONTAINERS_CONF_OVERRIDE logger options does not affect remote client"
 
     # This test makes sure that events are not returned more than once when
     # streaming during a log-file rotation.
@@ -243,7 +243,7 @@ events_logfile_max_size=11300
 EOF
 
     _populate_events_file $eventsFile
-    CONTAINERS_CONF=$containersConf timeout --kill=10 20 \
+    CONTAINERS_CONF_OVERRIDE=$containersConf timeout --kill=10 20 \
         $PODMAN events --stream=true --since="2022-03-06T11:26:42.723667984+02:00" --format=json > $eventsJSON &
 
     # Now wait for the above podman-events process to write to the eventsJSON
@@ -259,9 +259,9 @@ EOF
     assert $retries -gt 0 \
            "Timed out waiting for podman-events to start reading pre-existing events"
 
-    CONTAINERS_CONF=$containersConf run_podman create $IMAGE
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman create $IMAGE
     ctrID=$output
-    CONTAINERS_CONF=$containersConf run_podman rm -f $ctrID
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman rm -f $ctrID
 
     # Now wait for the last event above to be read by the `podman-events`
     # process above.
@@ -334,11 +334,11 @@ EOF
     local cname=c$(random_string 15)
     t0=$(date --iso-8601=seconds)
 
-    CONTAINERS_CONF=$containersConf run_podman create --name=$cname $IMAGE
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman create --name=$cname $IMAGE
-    CONTAINERS_CONF=$containersConf run_podman container inspect --size=true $cname
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman container inspect --size=true $cname
     inspect_json=$(jq -r --tab . <<< "$output")
 
-    CONTAINERS_CONF=$containersConf run_podman --events-backend=$1 events \
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman --events-backend=$1 events \
         --since="$t0"           \
         --filter=status=$cname  \
         --filter=status=create  \
@@ -349,7 +349,7 @@ EOF
 
     # Make sure that the inspect data doesn't show by default in
     # podman-events.
-    CONTAINERS_CONF=$containersConf run_podman --events-backend=$1 events \
+    CONTAINERS_CONF_OVERRIDE=$containersConf run_podman --events-backend=$1 events \
         --since="$t0"           \
         --filter=status=$cname  \
         --filter=status=create  \

test/system/160-volumes.bats

@@ -481,16 +481,16 @@ EOF
     run_podman 1 run --image-volume ignore --rm volume_image stat -f -c %T /data
     is "$output" "stat: can't read file system information for '/data': No such file or directory" "Should fail with /data does not exist"
 
-    CONTAINERS_CONF="$containersconf" run_podman run --rm volume_image stat -f -c %T /data
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman run --rm volume_image stat -f -c %T /data
     is "$output" "tmpfs" "Should be tmpfs"
 
-    CONTAINERS_CONF="$containersconf" run_podman run --image-volume bind --rm volume_image stat -f -c %T /data
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman run --image-volume bind --rm volume_image stat -f -c %T /data
     assert "$output" != "tmpfs" "Should match hosts $fs"
 
-    CONTAINERS_CONF="$containersconf" run_podman run --image-volume tmpfs --rm volume_image stat -f -c %T /data
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman run --image-volume tmpfs --rm volume_image stat -f -c %T /data
     is "$output" "tmpfs" "Should be tmpfs"
 
-    CONTAINERS_CONF="$containersconf" run_podman 1 run --image-volume ignore --rm volume_image stat -f -c %T /data
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman 1 run --image-volume ignore --rm volume_image stat -f -c %T /data
     is "$output" "stat: can't read file system information for '/data': No such file or directory" "Should fail with /data does not exist"
 
     run_podman rm --all --force -t 0

test/system/170-run-userns.bats

@@ -83,7 +83,7 @@ function _require_crun() {
 userns="auto"
 EOF
     # First make sure a user namespace is created
-    CONTAINERS_CONF=$PODMAN_TMPDIR/userns_auto.conf run_podman run -d $IMAGE sleep infinity
+    CONTAINERS_CONF_OVERRIDE=$PODMAN_TMPDIR/userns_auto.conf run_podman run -d $IMAGE sleep infinity
     cid=$output
 
     run_podman inspect --format '{{.HostConfig.UsernsMode}}' $cid
@@ -92,7 +92,7 @@ EOF
     run_podman rm -t 0 -f $cid
 
     # Then check that the main user is not mapped into the user namespace
-    CONTAINERS_CONF=$PODMAN_TMPDIR/userns_auto.conf run_podman 0 run --rm $IMAGE awk '{if($2 == "0"){exit 1}}' /proc/self/uid_map /proc/self/gid_map
+    CONTAINERS_CONF_OVERRIDE=$PODMAN_TMPDIR/userns_auto.conf run_podman 0 run --rm $IMAGE awk '{if($2 == "0"){exit 1}}' /proc/self/uid_map /proc/self/gid_map
 }
 
 @test "podman userns=auto and secrets" {

test/system/200-pod.bats

@@ -62,7 +62,7 @@ function teardown() {
 
 
 @test "podman pod create - custom infra image" {
-    skip_if_remote "CONTAINERS_CONF only affects server side"
+    skip_if_remote "CONTAINERS_CONF_OVERRIDE only affects server side"
     image="i.do/not/exist:image"
     tmpdir=$PODMAN_TMPDIR/pod-test
     mkdir -p $tmpdir
@@ -75,10 +75,10 @@ EOF
     run_podman 125 pod create --infra-image $image
     is "$output" ".*initializing source docker://$image:.*"
 
-    CONTAINERS_CONF=$containersconf run_podman 125 pod create
+    CONTAINERS_CONF_OVERRIDE=$containersconf run_podman 125 pod create
     is "$output" ".*initializing source docker://$image:.*"
 
-    CONTAINERS_CONF=$containersconf run_podman 125 create --pod new:test $IMAGE
+    CONTAINERS_CONF_OVERRIDE=$containersconf run_podman 125 create --pod new:test $IMAGE
     is "$output" ".*initializing source docker://$image:.*"
 }
 

test/system/500-networking.bats

@@ -630,8 +630,8 @@ load helpers.network
     run_podman network rm -t 0 -f $netname
 }
 
-@test "podman run CONTAINERS_CONF dns options" {
+@test "podman run CONTAINERS_CONF_OVERRIDE dns options" {
-    skip_if_remote "CONTAINERS_CONF redirect does not work on remote"
+    skip_if_remote "CONTAINERS_CONF_OVERRIDE redirect does not work on remote"
     # Test on the CLI and via containers.conf
     containersconf=$PODMAN_TMPDIR/containers.conf
 
@@ -650,7 +650,7 @@ EOF
     local nl="
 "
 
-    CONTAINERS_CONF=$containersconf run_podman run --rm $IMAGE cat /etc/resolv.conf
+    CONTAINERS_CONF_OVERRIDE=$containersconf run_podman run --rm $IMAGE cat /etc/resolv.conf
     is "$output" "search example.com.*" "correct search domain"
     is "$output" ".*nameserver 1.1.1.1${nl}nameserver $searchIP${nl}nameserver 1.0.0.1${nl}nameserver 8.8.8.8" "nameserver order is correct"
 
@@ -659,7 +659,7 @@ EOF
     local subnet=$(random_rfc1918_subnet)
     run_podman network create --subnet "$subnet.0/24"  $netname
     # custom server overwrites the network dns server
-    CONTAINERS_CONF=$containersconf run_podman run --network $netname --rm $IMAGE cat /etc/resolv.conf
+    CONTAINERS_CONF_OVERRIDE=$containersconf run_podman run --network $netname --rm $IMAGE cat /etc/resolv.conf
     is "$output" "search example.com.*" "correct search domain"
     local store=$output
     if is_netavark; then
@@ -714,8 +714,8 @@ nameserver 8.8.8.8" "nameserver order is correct"
     done
 }
 
-@test "podman run CONTAINERS_CONF /etc/hosts options" {
+@test "podman run CONTAINERS_CONF_OVERRIDE /etc/hosts options" {
-    skip_if_remote "CONTAINERS_CONF redirect does not work on remote"
+    skip_if_remote "CONTAINERS_CONF_OVERRIDE redirect does not work on remote"
 
     containersconf=$PODMAN_TMPDIR/containers.conf
     basehost=$PODMAN_TMPDIR/host
@@ -740,7 +740,7 @@ EOF
     ip3="$(random_rfc1918_subnet).$((RANDOM % 256))"
     name3=host3$(random_string)
 
-    CONTAINERS_CONF=$containersconf run_podman run --rm --add-host $name3:$ip3 $IMAGE cat /etc/hosts
+    CONTAINERS_CONF_OVERRIDE=$containersconf run_podman run --rm --add-host $name3:$ip3 $IMAGE cat /etc/hosts
     is "$output" ".*$ip3[[:blank:]]$name3.*" "--add-host entry in /etc/host"
     is "$output" ".*$ip1[[:blank:]]$name1.*" "first base entry in /etc/host"
     is "$output" ".*$ip2[[:blank:]]$name2.*" "second base entry in /etc/host"
@@ -751,7 +751,7 @@ EOF
 
     # now try again with container name and hostname == host entry name
     # in this case podman should not add its own entry thus we only have 5 entries (-1 for the removed --add-host)
-    CONTAINERS_CONF=$containersconf run_podman run --rm --name $name1 --hostname $name1 $IMAGE cat /etc/hosts
+    CONTAINERS_CONF_OVERRIDE=$containersconf run_podman run --rm --name $name1 --hostname $name1 $IMAGE cat /etc/hosts
     is "$output" ".*$ip1[[:blank:]]$name1.*" "first base entry in /etc/host"
     is "$output" ".*$ip2[[:blank:]]$name2.*" "second base entry in /etc/host"
     is "$output" ".*$containersinternal_ip[[:blank:]]host\.containers\.internal.*" "host.containers.internal ip from config in /etc/host"

test/system/700-play.bats

@@ -303,14 +303,14 @@ read_only=true
 EOF
 
     YAML=$PODMAN_TMPDIR/test.yml
-    CONTAINERS_CONF="$containersconf" run_podman create --pod new:pod1 --read-only=false --name test1 $IMAGE touch /testrw
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman create --pod new:pod1 --read-only=false --name test1 $IMAGE touch /testrw
-    CONTAINERS_CONF="$containersconf" run_podman create --pod pod1 --name test2 $IMAGE touch /testro
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman create --pod pod1 --name test2 $IMAGE touch /testro
-    CONTAINERS_CONF="$containersconf" run_podman create --pod pod1 --name test3 $IMAGE touch /tmp/testtmp
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman create --pod pod1 --name test3 $IMAGE touch /tmp/testtmp
-    CONTAINERS_CONF="$containersconf" run_podman container inspect --format '{{.HostConfig.ReadonlyRootfs}}' test1 test2 test3
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman container inspect --format '{{.HostConfig.ReadonlyRootfs}}' test1 test2 test3
     is "$output" "false.*true.*true" "Rootfs should be read/only"
 
     # Now generate and run kube.yaml on a machine without the defaults set
-    CONTAINERS_CONF="$containersconf" run_podman kube generate pod1 -f $YAML
+    CONTAINERS_CONF_OVERRIDE="$containersconf" run_podman kube generate pod1 -f $YAML
     cat $YAML
 
     run_podman kube play --replace $YAML

test/system/800-config.bats

@@ -13,12 +13,15 @@ load helpers
     runtime="$output"
     run_podman info --format "{{ .Host.OCIRuntime.Path }}"
     ocipath="$output"
+    run_podman info --format '{{ .Host.DatabaseBackend }}'
+    db_backend="$output"
 
     # Make an innocuous containers.conf in a non-standard location
     conf_tmp="$PODMAN_TMPDIR/containers.conf"
     cat >$conf_tmp <<EOF
 [engine]
 runtime="$runtime"
+database_backend="$db_backend"
 [engine.runtimes]
 $runtime = ["$ocipath"]
 EOF
@@ -43,11 +46,14 @@ EOF
     # Get the path of the normal runtime
     run_podman info --format "{{ .Host.OCIRuntime.Path }}"
     ocipath="$output"
+    run_podman info --format '{{ .Host.DatabaseBackend }}'
+    db_backend="$output"
 
     export conf_tmp="$PODMAN_TMPDIR/nonstandard_runtime_name.conf"
     cat > $conf_tmp <<EOF
 [engine]
 runtime = "nonstandard_runtime_name"
+database_backend="$db_backend"
 [engine.runtimes]
 nonstandard_runtime_name = ["$ocipath"]
 EOF