Cirrus: Add netavark/aardvark system test task

Also add a system-test that verifies netavark driver is in use when
magic env. var. is set.

Signed-off-by: Chris Evich <cevich@redhat.com>

.cirrus.yml

@@ -24,6 +24,11 @@ env:
     # Runner statistics log file path/name
     STATS_LOGFILE_SFX: 'runner_stats.log'
     STATS_LOGFILE: '$GOSRC/${CIRRUS_TASK_NAME}-${STATS_LOGFILE_SFX}'
+    # Netavark/aardvark location/options when TEST_ENVIRON=host-netavark
+    NETAVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/netavark/success/binary.zip?branch=${NETAVARK_BRANCH}"
+    NETAVARK_DEBUG: 0  # set non-zero to use the debug-mode binary
+    AARDVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/aardvark-dns/success/binary.zip?branch=${AARDVARK_BRANCH}"
+    AARDVARK_DEBUG: 0  # set non-zero to use the debug-mode binary
 
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
@@ -512,6 +517,7 @@ container_integration_test_task:
     main_script: *main
     always: *int_logs_artifacts
 
+
 # Run the integration tests using the latest upstream build of netavark.
 netavark_integration_test_task:
     name: "Netavark integration"  # using *std_name_fmt here is unreadable
@@ -528,10 +534,6 @@ netavark_integration_test_task:
         CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
         TEST_FLAVOR: int
         TEST_ENVIRON: host-netavark
-        NETAVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/netavark/success/binary.zip?branch=${NETAVARK_BRANCH}"
-        NETAVARK_DEBUG: 0  # set non-zero to use the debug-mode binary
-        AARDVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/aardvark-dns/success/binary.zip?branch=${AARDVARK_BRANCH}"
-        AARDVARK_DEBUG: 0  # set non-zero to use the debug-mode binary
     clone_script: *noop  # Comes from cache
     gopath_cache: *ro_gopath_cache
     setup_script: *setup
@@ -539,6 +541,28 @@ netavark_integration_test_task:
     always: *int_logs_artifacts
 
 
+netavark_system_test_task:
+    name: "Netavark system"
+    alias: netavark_system_test
+    skip: *tags
+    only_if: *not_build
+    depends_on:
+      - netavark_integration_test
+    gce_instance: *standardvm
+    env:
+        DISTRO_NV: ${FEDORA_NAME}
+        _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
+        VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
+        CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+        TEST_ENVIRON: host-netavark
+        TEST_FLAVOR: sys
+    clone_script: *noop  # Comes from cache
+    gopath_cache: *ro_gopath_cache
+    setup_script: *setup
+    main_script: *main
+    always: *logs_artifacts
+
+
 # Execute most integration tests as a regular (non-root) user.
 rootless_integration_test_task:
     name: *std_name_fmt
@@ -591,6 +615,7 @@ remote_system_test_task:
         TEST_FLAVOR: sys
         PODBIN_NAME: remote
 
+
 rootless_remote_system_test_task:
     <<: *local_system_test_task
     alias: rootless_remote_system_test
@@ -611,6 +636,7 @@ rootless_remote_system_test_task:
         PODBIN_NAME: remote
         PRIV_NAME: rootless
 
+
 buildah_bud_test_task:
     name: *std_name_fmt
     alias: buildah_bud_test
@@ -639,6 +665,7 @@ buildah_bud_test_task:
     main_script: *main
     always: *int_logs_artifacts
 
+
 rootless_system_test_task:
     name: *std_name_fmt
     alias: rootless_system_test
@@ -657,6 +684,7 @@ rootless_system_test_task:
     main_script: *main
     always: *logs_artifacts
 
+
 rootless_gitlab_test_task:
     name: *std_name_fmt
     alias: rootless_gitlab_test
@@ -684,6 +712,7 @@ rootless_gitlab_test_task:
             type: text/xml
             format: junit
 
+
 upgrade_test_task:
     name: "Upgrade test: from $PODMAN_UPGRADE_FROM"
     alias: upgrade_test
@@ -713,6 +742,7 @@ upgrade_test_task:
     main_script: *main
     always: *logs_artifacts
 
+
 # This task is critical.  It updates the "last-used by" timestamp stored
 # in metadata for all VM images.  This mechanism functions in tandem with
 # an out-of-band pruning operation to remove disused VM images.
@@ -761,10 +791,10 @@ success_task:
         - compose_test
         - local_integration_test
         - remote_integration_test
-        - netavark_integration_test
-        - rootless_integration_test
         - container_integration_test
         - netavark_integration_test
+        - netavark_system_test
+        - rootless_integration_test
         - local_system_test
         - remote_system_test
         - rootless_system_test

contrib/cirrus/lib.sh

@@ -214,16 +214,22 @@ setup_rootless() {
 }
 
 install_test_configs() {
-    echo "Installing cni config, policy and registry config"
+    msg "Installing ./test/registries.conf system-wide."
-    req_env_vars GOSRC SCRIPT_BASE
-    cd $GOSRC || exit 1
-    install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/
-    # This config must always sort last in the list of networks (podman picks first one
-    # as the default).  This config prevents allocation of network address space used
-    # by default in google cloud.  https://cloud.google.com/vpc/docs/vpc#ip-ranges
-    install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist /etc/cni/net.d/
-
     install -v -D -m 644 ./test/registries.conf /etc/containers/
+    if [[ "$TEST_ENVIRON" =~ netavark ]]; then
+        # belt-and-suspenders: any pre-existing CNI config. will spoil
+        # default use tof netavark (when both are installed).
+        rm -rf /etc/cni/net.d/*
+    else
+        echo "Installing cni config, policy and registry config"
+        req_env_vars GOSRC SCRIPT_BASE
+        cd $GOSRC || exit 1
+        install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/
+        # This config must always sort last in the list of networks (podman picks first one
+        # as the default).  This config prevents allocation of network address space used
+        # by default in google cloud.  https://cloud.google.com/vpc/docs/vpc#ip-ranges
+        install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist /etc/cni/net.d/
+    fi
 }
 
 # Remove all files provided by the distro version of podman.

contrib/cirrus/setup_environment.sh

@@ -173,6 +173,8 @@ case "$TEST_ENVIRON" in
             done
 
             restorecon -F -v $_pdir
+            # This is critical, it signals to all tests that netavark
+            # use is expected.
             msg "Forcing NETWORK_BACKEND=netavark in all subsequent environments."
             echo "NETWORK_BACKEND=netavark" >> /etc/ci_environment
         fi

test/system/005-info.bats

@@ -88,6 +88,18 @@ host.slirp4netns.executable | $expr_path
     is "$output" ".*graphOptions: {}" "output includes graphOptions: {}"
 }
 
+@test "podman info netavark " {
+    # Confirm netavark in use when explicitely required by execution environment.
+    if [[ "$NETWORK_BACKEND" == "netavark" ]]; then
+        if ! is_netavark; then
+            # Assume is_netavark() will provide debugging feedback.
+            die "Netavark driver testing required, but not in use by podman."
+        fi
+    else
+        skip "Netavark testing not requested (\$NETWORK_BACKEND='$NETWORK_BACKEND')"
+    fi
+}
+
 @test "podman --root PATH info - basic output" {
     if ! is_remote; then
         run_podman --storage-driver=vfs --root ${PODMAN_TMPDIR}/nothing-here-move-along info --format '{{ .Store.GraphOptions }}'

test/system/500-networking.bats

@@ -256,13 +256,17 @@ load helpers
 
     # rootless cannot modify iptables
     if ! is_rootless; then
-        # flush the CNI iptables here
+        # flush the port forwarding iptable rule here
-        run iptables -t nat -F CNI-HOSTPORT-DNAT
+        chain="CNI-HOSTPORT-DNAT"
+        if is_netavark; then
+            chain="NETAVARK-HOSTPORT-DNAT"
+        fi
+        run iptables -t nat -F "$chain"
 
         # check that we cannot curl (timeout after 5 sec)
         run timeout 5 curl -s $SERVER/index.txt
         if [ "$status" -ne 124 ]; then
-	        die "curl did not timeout, status code: $status"
+            die "curl did not timeout, status code: $status"
         fi
     fi
 

test/system/README.md

@@ -49,6 +49,7 @@ Running tests
 To run the tests locally in your sandbox, you can use one of these methods:
 * make;PODMAN=./bin/podman bats ./test/system/070-build.bats # runs just the specified test
 * make;PODMAN=./bin/podman bats ./test/system                # runs all
+* make;PODMAN=./bin/podman NETWORK_BACKEND=netavark bats ./test/system  # Assert & enable netavark testing
 
 To test as root:
 *  $ PODMAN=./bin/podman sudo --preserve-env=PODMAN bats test/system

test/system/helpers.bash

@@ -341,6 +341,15 @@ function is_cgroupsv2() {
     test "$cgroup_type" = "cgroup2fs"
 }
 
+# True if podman is using netavark
+function is_netavark() {
+    run_podman info --format '{{.Host.NetworkBackend}}'
+    if [[ "$output" =~ netavark ]]; then
+        return 0
+    fi
+    return 1
+}
+
 # Returns the OCI runtime *basename* (typically crun or runc). Much as we'd
 # love to cache this result, we probably shouldn't.
 function podman_runtime() {