Move all python tests to pytest

* Add configuration to add report header for python client used in tests
* Move report headers into the individual test runners vs runner.sh

Signed-off-by: Jhon Honce <jhonce@redhat.com>

Makefile

@@ -542,7 +542,7 @@ validate.completions:
 run-docker-py-tests:
 	touch test/__init__.py
 	env CONTAINERS_CONF=$(CURDIR)/test/apiv2/containers.conf pytest --disable-warnings test/python/docker/
-	-rm test/__init__.py
+	rm -f test/__init__.py
 
 .PHONY: localunit
 localunit: test/goecho/goecho test/version/version
@@ -623,9 +623,15 @@ remotesystem:
 
 .PHONY: localapiv2
 localapiv2:
-	env PODMAN=./bin/podman ./test/apiv2/test-apiv2
-	env CONTAINERS_CONF=$(CURDIR)/test/apiv2/containers.conf PODMAN=./bin/podman ${PYTHON} -m unittest discover -v ./test/apiv2/python
-	env CONTAINERS_CONF=$(CURDIR)/test/apiv2/containers.conf PODMAN=./bin/podman ${PYTHON} -m unittest discover -v ./test/python/docker
+	# Order is important running python tests first causes the bash tests to fail, see 12-imagesMore
+	# FIXME order of tests should not matter
+	env PODMAN=./bin/podman stdbuf -o0 -e0 ./test/apiv2/test-apiv2
+	env CONTAINERS_CONF=$(CURDIR)/test/apiv2/containers.conf PODMAN=./bin/podman \
+		pytest --disable-warnings ./test/apiv2/python
+	touch test/__init__.py
+	env CONTAINERS_CONF=$(CURDIR)/test/apiv2/containers.conf PODMAN=./bin/podman \
+		pytest --disable-warnings ./test/python/docker
+	rm -f test/__init__.py
 
 .PHONY: remoteapiv2
 remoteapiv2:
@@ -949,5 +955,5 @@ clean: clean-binaries ## Clean all make artifacts
 		libpod/pod_easyjson.go \
 		.install.goimports \
 		docs/build \
-		venv
+		.venv
 	make -C docs clean

contrib/cirrus/runner.sh

@@ -55,11 +55,8 @@ function _run_unit() {
 }
 
 function _run_apiv2() {
-    local m="Testing of API was performed using the **PODMAN*** client"
-    warn "$m"
-    if ! make localapiv2 |& logformatter; then
-        die "$m"
-    fi
+    source .venv/requests/bin/activate
+    make localapiv2 |& logformatter
 }
 
 function _run_compose() {
@@ -100,12 +97,8 @@ function _run_bindings() {
 }
 
 function _run_docker-py() {
-    m="Testing of API was performed using the **DOCKER** client"
-    warn "$m"
-    source venv/bin/activate
-    if ! make run-docker-py-tests; then
-        die "$m"
-    fi
+    source .venv/docker-py/bin/activate
+    make run-docker-py-tests
 }
 
 function _run_endpoint() {

contrib/cirrus/setup_environment.sh

@@ -265,14 +265,21 @@ case "$TEST_FLAVOR" in
 
         msg "Installing previously downloaded/cached packages"
         dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
-        virtualenv venv
-        source venv/bin/activate
+        virtualenv .venv/docker-py
+        source .venv/docker-py/bin/activate
         pip install --upgrade pip
         pip install --requirement $GOSRC/test/python/requirements.txt
         ;;
     build) make clean ;;
     unit) ;;
-    apiv2) ;&  # use next item
+    apiv2)
+        msg "Installing previously downloaded/cached packages"
+        dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
+        virtualenv .venv/requests
+        source .venv/requests/bin/activate
+        pip install --upgrade pip
+        pip install --requirement $GOSRC/test/apiv2/python/requirements.txt
+        ;&  # continue with next item
     compose)
         rpm -ivh $PACKAGE_DOWNLOAD_DIR/podman-docker*
         ;&  # continue with next item

test/apiv2/12-imagesMore.at

@@ -26,7 +26,8 @@ t GET libpod/images/$IMAGE/json 200 \
   .RepoTags[1]=localhost:5000/myrepo:mytag
 
 # Run registry container
-podman run -d --name registry -p 5000:5000 quay.io/libpod/registry:2.6 /entrypoint.sh /etc/docker/registry/config.yml
+# FIXME this fails if python tests have been run first...
+podman run -d --name registry -p 5000:5000 quay.io/libpod/registry:2.7 /entrypoint.sh /etc/docker/registry/config.yml
 wait_for_port localhost 5000
 
 # Push to local registry and check output
@@ -58,7 +59,7 @@ t DELETE libpod/containers/registry?force=true 200
 # Remove images
 t DELETE libpod/images/$IMAGE 200 \
   .ExitCode=0
-t DELETE libpod/images/quay.io/libpod/registry:2.6 200 \
+t DELETE libpod/images/quay.io/libpod/registry:2.7 200 \
   .ExitCode=0
 
 if [ -z "${GOT_DIGEST}" ] ; then

test/apiv2/python/conftest.py

@@ -0,0 +1,8 @@
+"""
+Configure pytest
+"""
+
+
+def pytest_report_header(config):
+    """Add header to report."""
+    return "python client -- requests library"

test/apiv2/python/requirements.txt

@@ -0,0 +1,5 @@
+requests-mock~=1.9.3
+requests~=2.20.0
+setuptools~=50.3.2
+python-dateutil~=2.8.1
+PyYAML~=5.4.1

test/apiv2/test-apiv2

@@ -368,7 +368,7 @@ function start_service() {
         die "Cannot start service on non-localhost ($HOST)"
     fi
 
-    echo $WORKDIR
+    echo "rootdir: "$WORKDIR
     # Some tests use shortnames; force registry override to work around
     # docker.io throttling.
 #    FIXME esm revisit pulling expected images re: shortnames caused tests to fail
@@ -376,7 +376,7 @@ function start_service() {
     $PODMAN_BIN \
         --root $WORKDIR/server_root --syslog=true \
         system service \
-        --time 15 \
+        --time 0 \
         tcp:127.0.0.1:$PORT \
         &> $WORKDIR/server.log &
     service_pid=$!
@@ -443,7 +443,7 @@ function start_registry() {
            -e REGISTRY_HTTP_TLS_KEY=/auth/domain.key \
            ${REGISTRY_IMAGE}
 
-    wait_for_port localhost $REGISTRY_PORT
+    wait_for_port localhost $REGISTRY_PORT 10
 }
 
 function stop_registry() {
@@ -492,13 +492,16 @@ function wait_for_port() {
     local port=$2                      # Numeric port
     local _timeout=${3:-5}             # Optional; default to 5 seconds
 
+    local path=/dev/tcp/$host/$port
+
     # Wait
-    while [ $_timeout -gt 0 ]; do
+    local i=$_timeout
+    while [ $i -gt 0 ]; do
         { exec 3<> /dev/tcp/$host/$port; } &>/dev/null && return
         sleep 1
-        _timeout=$(( $_timeout - 1 ))
+        i=$(( $i - 1 ))
     done
-    die "Timed out waiting for service"
+    die "Timed out (${_timeout}s) waiting for service ($path)"
 }
 
 ############
@@ -543,6 +546,9 @@ done
 ###############################################################################
 # BEGIN entry handler (subtest invoker)
 
+echo '============================= test session starts =============================='
+echo "podman client -- $(curl --version)"
+
 # Identify the tests to run. If called with args, use those as globs.
 tests_to_run=()
 if [ -n "$*" ]; then
@@ -558,6 +564,7 @@ if [ -n "$*" ]; then
 else
     tests_to_run=($TESTS_DIR/*.at)
 fi
+echo -e "collected ${#tests_to_run[@]} items\n"
 
 start_service