CI: APIv2 tests: add rootless

This would've caught a regression that #14549 had to fix.
Let's try to prevent the next regression.

This requires some hackery to get namespaces initialized
before the service is started; otherwise the service itself
initializes namespaces, which basically ends up with a
server process that runs forever.

Also: in stop_service(), reset service_pid, because that's
the correct thing to do.

Also: add some debug statements to try to figure out a
CI failure. (And leave them in place, because they might
be useful for future problems).

Signed-off-by: Ed Santiago <santiago@redhat.com>

.cirrus.yml

@@ -431,7 +431,7 @@ unit_test_task:
 
 
 apiv2_test_task:
-    name: "APIv2 test on $DISTRO_NV"
+    name: "APIv2 test on $DISTRO_NV ($PRIV_NAME)"
     alias: apiv2_test
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *not_tag_branch_build_docs
@@ -445,6 +445,11 @@ apiv2_test_task:
     env:
         <<: *stdenvars
         TEST_FLAVOR: apiv2
+    matrix:
+      - env:
+          PRIV_NAME: root
+      - env:
+          PRIV_NAME: rootless
     clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main

test/apiv2/test-apiv2

@@ -380,6 +380,17 @@ function start_service() {
         die "Cannot start service on non-localhost ($HOST)"
     fi
 
+    # FIXME: EXPERIMENTAL: 2022-06-13: podman rootless needs a namespace. If
+    # system-service is the first podman command run (as is the case in CI)
+    # this will happen as a fork-exec, where the parent podman creates the
+    # namespace and the child is the server. Then, when stop_service() kills
+    # the parent, the child (server) happily stays alive and ruins subsequent
+    # tests that try to restart service with different settings.
+    # Workaround: run an unshare to get namespaces initialized.
+    if [[ $(id -u) != 0 ]]; then
+        $PODMAN_BIN unshare true
+    fi
+
     $PODMAN_BIN \
         --root $WORKDIR/server_root --syslog=true \
         system service \
@@ -387,6 +398,7 @@ function start_service() {
         tcp:127.0.0.1:$PORT \
         &> $WORKDIR/server.log &
     service_pid=$!
+    echo "# started service, pid $service_pid"
 
     wait_for_port $HOST $PORT
 }
@@ -396,7 +408,14 @@ function stop_service() {
     if [[ -n $service_pid ]]; then
         kill $service_pid || :
         wait $service_pid || :
+        echo "# stopped service, pid $service_pid"
     fi
+    service_pid=
+
+    if { exec 3<> /dev/tcp/$HOST/$PORT; } &>/dev/null; then
+        echo "# WARNING: stop_service: Service still running on port $PORT"
+    fi
+
 }
 
 ####################
@@ -468,6 +487,7 @@ function start_registry() {
            ${REGISTRY_IMAGE}
 
     wait_for_port localhost $REGISTRY_PORT 10
+    echo "# started registry (auth=$auth) on port $PORT"
 }
 
 function stop_registry() {
@@ -482,6 +502,7 @@ function stop_registry() {
         if [[ "$1" = "--cleanup" ]]; then
             podman $OPTS rmi -f -a
         fi
+        echo "# stopped registry on port $PORT"
     fi
 
     REGISTRY_PORT=