add hack/perf for comparing two container engines

Add a set of scripts using hyperfine for comparing two container
engines.  I am currently using the scripts for comparing Podman
and Docker, and with older versions of Podman.

These scripts are not meant for production usage but to aid in tracking
down performance regressions and bottlenecks.

Run the scripts via `sudo sh $script.sh`.

Use the following environment variables to change the default behavior:
* `ENGINE_A` to set container engine A (default `/usr/bin/podman`)
* `ENGINE_B` to set container engine B (default `/usr/bin/docker`)
* `RUNS` to change the runs/repetitions of each benchmarks (default `100`)
* `NUM_CONTAINERS` to change the number of created containers for some benchmarks (e.g., `ps`) (default `100`)
* `IMAGE` to change the default container image (default `docker.io/library/alpine:latest`)

Signed-off-by: Valentin Rothberg <vrothberg@redhat.com>

hack/perf/README.md

@@ -0,0 +1,12 @@
+# A set of scripts to compare the performance of two container engines
+
+Run the scripts via `sudo sh $script.sh`.
+
+WARNING: Running any script will run `systemd prune`.
+
+Use the following environment variables to change the default behavior:
+* `ENGINE_A` to set container engine A (default `podman`)
+* `ENGINE_B` to set container engine B (default `docker`)
+* `RUNS` to change the runs/repetitions of each benchmarks (default `100`)
+* `NUM_CONTAINERS` to change the number of created containers for some benchmarks (e.g., `ps`) (default `100`)
+* `IMAGE` to change the default container image (default `docker.io/library/alpine:latest`)

hack/perf/create.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+source ./helpers.bash
+
+setup
+echo_bold "Create $RUNS containers"
+hyperfine --warmup 10 --runs $RUNS \
+	"$ENGINE_A create $IMAGE" \
+	"$ENGINE_B create $IMAGE"

hack/perf/helpers.bash

@@ -0,0 +1,36 @@
+ENGINE_A=${ENGINE_A:-podman}
+ENGINE_B=${ENGINE_B:-docker}
+RUNS=${RUNS:-100}
+NUM_CONTAINERS=${NUM_CONTAINERS:-100}
+IMAGE=${IMAGE:-docker.io/library/alpine:latest}
+
+BOLD="$(tput bold)"
+RESET="$(tput sgr0)"
+
+function echo_bold() {
+    echo "${BOLD}$1${RESET}"
+}
+
+function pull_image() {
+	echo_bold "... pulling $IMAGE"
+	$ENGINE_A pull $IMAGE -q > /dev/null
+	$ENGINE_B pull $IMAGE -q > /dev/null
+}
+
+function setup() {
+	echo_bold "---------------------------------------------------"
+	echo_bold "... comparing $ENGINE_A with $ENGINE_B"
+	echo_bold "... cleaning up previous containers and images"
+	$ENGINE_A system prune -f > /dev/null
+	$ENGINE_B system prune -f > /dev/null
+	pull_image
+	echo ""
+}
+
+function create_containers() {
+	echo_bold "... creating $NUM_CONTAINERS containers"
+	for i in $(eval echo "{0..$NUM_CONTAINERS}"); do
+		$ENGINE_A create $IMAGE >> /dev/null
+		$ENGINE_B create $IMAGE >> /dev/null
+	done
+}

hack/perf/ps.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+source ./helpers.bash
+
+setup
+echo_bold "List $NUM_CONTAINERS created containers"
+create_containers
+hyperfine --warmup 10 --runs $RUNS \
+	"$ENGINE_A ps -a" \
+	"$ENGINE_B ps -a"

hack/perf/rm.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+source ./helpers.bash
+
+setup
+echo_bold "Remove $RUNS containers in a row"
+hyperfine --warmup 10 --runs $RUNS \
+	--prepare "$ENGINE_A create --name=123 $IMAGE" \
+	--prepare "$ENGINE_B create --name=123 $IMAGE" \
+	"$ENGINE_A rm 123" \
+	"$ENGINE_B rm 123"

hack/perf/run.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env sh
+source ./helpers.bash
+
+setup
+echo_bold "Run $RUNS containers in a row"
+hyperfine --warmup 10 --runs $RUNS \
+	--prepare "$ENGINE_A rm -f 123 || true" \
+	--prepare "$ENGINE_B rm -f 123 || true" \
+	"$ENGINE_A run --name=123 $IMAGE true" \
+	"$ENGINE_B run --name=123 $IMAGE true"
+
+setup
+echo_bold "Run and remove $RUNS containers in a row"
+hyperfine --warmup 10 --runs $RUNS \
+	--prepare "$ENGINE_A rm -f 123 || true" \
+	--prepare "$ENGINE_B rm -f 123 || true" \
+	"$ENGINE_A run --rm --name=123 $IMAGE true" \
+	"$ENGINE_B run --rm --name=123 $IMAGE true"

hack/perf/start.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+source ./helpers.bash
+
+setup
+echo_bold "Start $RUNS containers in a row"
+hyperfine --warmup 10 --runs $RUNS \
+	--prepare "$ENGINE_A rm -f 123 || true; $ENGINE_A create --name=123 $IMAGE true" \
+	--prepare "$ENGINE_B rm -f 123 || true; $ENGINE_B create --name=123 $IMAGE true" \
+	"$ENGINE_A start 123" \
+	"$ENGINE_B start 123"

hack/perf/stop.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+source ./helpers.bash
+
+setup
+echo_bold "Stop $RUNS containers in a row"
+hyperfine --warmup 10 --runs $RUNS \
+	--prepare "$ENGINE_A rm -f 123 || true; $ENGINE_A run -d --name=123 $IMAGE top" \
+	--prepare "$ENGINE_B rm -f 123 || true; $ENGINE_B run -d --name=123 $IMAGE top" \
+	"$ENGINE_A stop 123" \
+	"$ENGINE_B stop 123"