Cirrus: Test w/ netavark/aardvark-dns in F36+

Now that netavark and aardvark are packaged and default in F36, support
CNI-based testing in F35 and Ubuntu.

* Remove the temporary/special `$TEST_ENVIRON=host-netavark` construct.
* Remove dedicated/special integration and system testing tasks.
* Update test-config setup to properly handle CNI vs netavark/aardvark
  environments.
* Update package-version logging to operate based on installed packages
  (along with some other minor script cleanups).
* Update global environment setup to force `$NETWORK_BACKEND=netavark`
  in F36 and later.  Except when `upgrade_test` task runs.
* Discontinue installing netavark and aardvark-dns binaries from
  upstream build artifacts.
* Drop CGV1-vs-2 policy check.  Ubuntu VMs now exclusively test CGv1,
  Fedora VMs test CGv2, with F35 testing CNI and F36 testing Netavark.

Signed-off-by: Chris Evich <cevich@redhat.com>

.cirrus.yml

@@ -7,10 +7,6 @@ env:
     ####
     # Name of the ultimate destination branch for this CI run, PR or post-merge.
     DEST_BRANCH: "main"
-    # Netavark branch to use when TEST_ENVIRON=host-netavark
-    NETAVARK_BRANCH: "main"
-    # Aardvark branch to use
-    AARDVARK_BRANCH: "main"
     # Overrides default location (/tmp/cirrus) for repo clone
     GOPATH: &gopath "/var/tmp/go"
     GOBIN: "${GOPATH}/bin"
@@ -24,11 +20,6 @@ env:
     # Runner statistics log file path/name
     STATS_LOGFILE_SFX: 'runner_stats.log'
     STATS_LOGFILE: '$GOSRC/${CIRRUS_TASK_NAME}-${STATS_LOGFILE_SFX}'
-    # Netavark/aardvark location/options when TEST_ENVIRON=host-netavark
-    NETAVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/netavark/success/binary.zip?branch=${NETAVARK_BRANCH}"
-    NETAVARK_DEBUG: 0  # set non-zero to use the debug-mode binary
-    AARDVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/aardvark-dns/success/binary.zip?branch=${AARDVARK_BRANCH}"
-    AARDVARK_DEBUG: 0  # set non-zero to use the debug-mode binary
 
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
@@ -53,7 +44,7 @@ env:
     #### N/B: Required ALL of these are set for every single task.
     ####
     TEST_FLAVOR:             # int, sys, ext_svc, validate, automation, etc.
-    TEST_ENVIRON: host       # 'host', 'host-netavark', or 'container'
+    TEST_ENVIRON: host       # 'host', or 'container'
     PODBIN_NAME: podman      # 'podman' or 'remote'
     PRIV_NAME: root          # 'root' or 'rootless'
     DISTRO_NV:               # any {PRIOR_,}{FEDORA,UBUNTU}_NAME value
@@ -553,41 +544,6 @@ rootless_integration_test_task:
     always: *int_logs_artifacts
 
 
-# Run various scenarios using upstream netavark/aardvark-dns binaries
-netavark_task:
-    name: "Netavark $TEST_FLAVOR $PODBIN_NAME $PRIV_NAME"
-    alias: netavark
-    only_if: *not_build
-    skip: *branches_and_tags
-    depends_on:
-        - unit_test
-    gce_instance: *standardvm
-    matrix:
-        - env: &nenv
-            DISTRO_NV: ${FEDORA_NAME}
-            _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
-            VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
-            CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
-            TEST_FLAVOR: int
-            TEST_ENVIRON: host-netavark
-        - env:
-            <<: *nenv
-            TEST_FLAVOR: int
-            PRIV_NAME: rootless
-        - env:
-            <<: *nenv
-            TEST_FLAVOR: sys
-        - env:
-            <<: *nenv
-            TEST_FLAVOR: sys
-            PRIV_NAME: rootless
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
-    setup_script: *setup
-    main_script: *main
-    always: *int_logs_artifacts
-
-
 # Always run subsequent to integration tests.  While parallelism is lost
 # with runtime, debugging system-test failures can be more challenging
 # for some golang developers.  Otherwise the following tasks run across
@@ -841,7 +797,6 @@ success_task:
         - remote_integration_test
         - container_integration_test
         - rootless_integration_test
-        - netavark
         - local_system_test
         - remote_system_test
         - rootless_system_test

contrib/cirrus/lib.sh

@@ -97,7 +97,7 @@ EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA"
 # testing operations on all platforms and versions.  This is necessary
 # to avoid needlessly passing through global/system values across
 # contexts, such as host->container or root->rootless user
-PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)'
+PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)|(NETWORK_BACKEND)'
 # Unsafe env. vars for display
 SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)|(PASSWORD)|(TOKEN)'
 
@@ -216,20 +216,34 @@ setup_rootless() {
 install_test_configs() {
     msg "Installing ./test/registries.conf system-wide."
     install -v -D -m 644 ./test/registries.conf /etc/containers/
-    if [[ "$TEST_ENVIRON" =~ netavark ]]; then
+}
-        # belt-and-suspenders: any pre-existing CNI config. will spoil
+
-        # default use tof netavark (when both are installed).
+use_cni() {
-        rm -rf /etc/cni/net.d/*
+    msg "Unsetting NETWORK_BACKEND for all subsequent environments."
-    else
+    echo "export -n NETWORK_BACKEND" >> /etc/ci_environment
-        echo "Installing cni config, policy and registry config"
+    echo "unset NETWORK_BACKEND" >> /etc/ci_environment
-        req_env_vars GOSRC SCRIPT_BASE
+    export -n NETWORK_BACKEND
-        cd $GOSRC || exit 1
+    unset NETWORK_BACKEND
-        install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/
+    msg "Installing default CNI configuration"
-        # This config must always sort last in the list of networks (podman picks first one
+    cd $GOSRC || exit 1
-        # as the default).  This config prevents allocation of network address space used
+    rm -rvf /etc/cni/net.d
-        # by default in google cloud.  https://cloud.google.com/vpc/docs/vpc#ip-ranges
+    mkdir -p /etc/cni/net.d
-        install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist /etc/cni/net.d/
+    install -v -D -m 644 ./cni/87-podman-bridge.conflist \
-    fi
+        /etc/cni/net.d/
+    # This config must always sort last in the list of networks (podman picks
+    # first one as the default).  This config prevents allocation of network
+    # address space used by default in google cloud.
+    # https://cloud.google.com/vpc/docs/vpc#ip-ranges
+    install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist \
+        /etc/cni/net.d/
+}
+
+use_netavark() {
+    msg "Forcing NETWORK_BACKEND=netavark for all subsequent environments."
+    echo "NETWORK_BACKEND=netavark" >> /etc/ci_environment
+    export NETWORK_BACKEND=netavark  # needed for install_test_configs()
+    msg "Removing any/all CNI configuration"
+    rm -rvf /etc/cni/net.d/*
 }
 
 # Remove all files provided by the distro version of podman.

contrib/cirrus/logcollector.sh

@@ -40,32 +40,34 @@ case $1 in
     packages)
         # These names are common to Fedora and Ubuntu
         PKG_NAMES=(\
-                    conmon \
+                    conmon
-                    containernetworking-plugins \
+                    containernetworking-plugins
-                    containers-common \
+                    containers-common
-                    criu \
+                    criu
-                    crun \
+                    crun
-                    golang \
+                    golang
-                    podman \
+                    podman
-                    runc \
+                    runc
-                    skopeo \
+                    skopeo
-                    slirp4netns \
+                    slirp4netns
         )
         case $OS_RELEASE_ID in
             fedora)
                 cat /etc/fedora-release
                 PKG_LST_CMD='rpm -q --qf=%{N}-%{V}-%{R}-%{ARCH}\n'
                 PKG_NAMES+=(\
-                    container-selinux \
+                    aardvark
-                    libseccomp \
+                    container-selinux
+                    libseccomp
+                    netavark
                 )
                 ;;
             ubuntu)
                 cat /etc/issue
                 PKG_LST_CMD='dpkg-query --show --showformat=${Package}-${Version}-${Architecture}\n'
                 PKG_NAMES+=(\
-                    cri-o-runc \
+                    cri-o-runc
-                    libseccomp2 \
+                    libseccomp2
                 )
                 ;;
             *) bad_os_id_ver ;;
@@ -74,19 +76,6 @@ case $1 in
         echo "Cgroups: " $(stat -f -c %T /sys/fs/cgroup)
         # Any not-present packages will be listed as such
         $PKG_LST_CMD "${PKG_NAMES[@]}" | sort -u
-
-        # TODO: Remove this once netavark/aardvark-dns packages are used
-        if [[ "$TEST_ENVIRON" =~ netavark ]]; then
-            _npath=/usr/local/libexec/podman/
-            for name in netavark aardvark-dns; do
-                echo "$name binary details:"
-                if [[ -r "$_npath/${name}.info" ]]; then
-                    cat "$_npath/${name}.info"
-                else
-                    echo "WARNING: $_npath/${name}.info not found."
-                fi
-            done
-        fi
         ;;
     time)
         # Assumed to be empty/undefined outside of Cirrus-CI (.cirrus.yml)

contrib/cirrus/runner.sh

@@ -12,7 +12,7 @@ set -eo pipefail
 # most notably:
 #
 #    PODBIN_NAME  : "podman" (i.e. local) or "remote"
-#    TEST_ENVIRON : 'host', 'host-netavark', or 'container'; desired environment in which to run
+#    TEST_ENVIRON : 'host', or 'container'; desired environment in which to run
 #    CONTAINER    : 1 if *currently* running inside a container, 0 if host
 #
 

contrib/cirrus/setup_environment.sh

@@ -38,9 +38,6 @@ done
 
 cp hack/podman-registry /bin
 
-# Make sure cni network plugins directory exists
-mkdir -p /etc/cni/net.d
-
 # Some test operations & checks require a git "identity"
 _gc='git config --file /root/.gitconfig'
 $_gc user.email "TMcTestFace@example.com"
@@ -84,13 +81,6 @@ case "$CG_FS_TYPE" in
             else
                 echo "OCI_RUNTIME=runc" >> /etc/ci_environment
             fi
-
-            # As a general policy CGv1 + runc should coincide with the "older"
-            # VM Images in CI.  Verify this is the case.
-            if [[ -n "$VM_IMAGE_NAME" ]] && [[ ! "$VM_IMAGE_NAME" =~ prior ]]
-            then
-                die "Most recent distro. version should never run with CGv1"
-            fi
         fi
         ;;
     cgroup2fs)
@@ -99,13 +89,6 @@ case "$CG_FS_TYPE" in
             # which uses runc as the default.
             warn "Forcing testing with crun instead of runc"
             echo "OCI_RUNTIME=crun" >> /etc/ci_environment
-
-            # As a general policy CGv2 + crun should coincide with the "newer"
-            # VM Images in CI.  Verify this is the case.
-            if [[ -n "$VM_IMAGE_NAME" ]] && [[ "$VM_IMAGE_NAME" =~ prior ]]
-            then
-                die "Least recent distro. version should never run with CGv2"
-            fi
         fi
         ;;
     *) die_unknown CG_FS_TYPE
@@ -130,6 +113,19 @@ case "$OS_RELEASE_ID" in
             msg "Enabling container_manage_cgroup"
             setsebool container_manage_cgroup true
         fi
+
+        # For release 36 and later, netavark/aardvark is the default
+        # networking stack for podman.  All previous releases only have
+        # CNI networking available.  Upgrading from one to the other is
+        # not supported at this time.  Support execution of the upgrade
+        # tests in F36 and later, by disabling Netavark and enabling CNI.
+        if [[ "$OS_RELEASE_VER" -ge 36 ]] && \
+           [[ "$TEST_FLAVOR" != "upgrade_test" ]];
+        then
+            use_netavark
+        else # Fedora < 36, or upgrade testing.
+            use_cni
+        fi
         ;;
     *) die_unknown OS_RELEASE_ID
 esac
@@ -137,7 +133,7 @@ esac
 # Required to be defined by caller: The environment where primary testing happens
 # shellcheck disable=SC2154
 case "$TEST_ENVIRON" in
-    host*)
+    host)
         # The e2e tests wrongly guess `--cgroup-manager` option
         # shellcheck disable=SC2154
         if [[ "$CG_FS_TYPE" == "cgroup2fs" ]] || [[ "$PRIV_NAME" == "root" ]]
@@ -148,43 +144,6 @@ case "$TEST_ENVIRON" in
             warn "Forcing CGROUP_MANAGER=cgroupfs"
             echo "CGROUP_MANAGER=cgroupfs" >> /etc/ci_environment
         fi
-        # TODO: For the foreseeable future, need to support running tests
-        # with and without the latest netavark/aardvark.  Once they're more
-        # stable and widely supported in Fedora, they can be pre-installed
-        # from its RPM at VM image build-time.
-        if [[ "$TEST_ENVIRON" =~ netavark ]]; then
-            for info in "netavark $NETAVARK_BRANCH $NETAVARK_URL $NETAVARK_DEBUG" \
-                        "aardvark-dns $AARDVARK_BRANCH $AARDVARK_URL $AARDVARK_DEBUG"; do
-
-                read _name _branch _url _debug <<<"$info"
-                req_env_vars _name _branch _url _debug
-                msg "Downloading latest $_name from upstream branch '$_branch'"
-                # Use identifiable archive filename in of a get_ci_env.sh environment
-                curl --fail --location -o /tmp/$_name.zip "$_url"
-
-                # Needs to be in a specific location
-                # ref: https://github.com/containers/common/blob/main/pkg/config/config_linux.go#L39
-                _pdir=/usr/local/libexec/podman
-                mkdir -p $_pdir
-                cd $_pdir
-                msg "$PWD"
-                unzip /tmp/$_name.zip
-                if ((_debug)); then
-                    warn "Using debug $_name binary"
-                    mv $_name.debug $_name
-                else
-                    rm $_name.debug
-                fi
-                chmod 0755 $_pdir/$_name
-                cd -
-            done
-
-            restorecon -F -v $_nvdir
-            # This is critical, it signals to all tests that netavark
-            # use is expected.
-            msg "Forcing NETWORK_BACKEND=netavark in all subsequent environments."
-            echo "NETWORK_BACKEND=netavark" >> /etc/ci_environment
-        fi
         ;;
     container)
         if ((CONTAINER==0)); then  # not yet inside a container