From 4ed46c9847daea72c9ed440882023843b12652c3 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 23 Jan 2023 14:01:57 +0100 Subject: [PATCH] add hack/perf for comparing two container engines Add a set of scripts using hyperfine for comparing two container engines. I am currently using the scripts for comparing Podman and Docker, and with older versions of Podman. These scripts are not meant for production usage but to aid in tracking down performance regressions and bottlenecks. Run the scripts via `sudo sh $script.sh`. Use the following environment variables to change the default behavior: * `ENGINE_A` to set container engine A (default `/usr/bin/podman`) * `ENGINE_B` to set container engine B (default `/usr/bin/docker`) * `RUNS` to change the runs/repetitions of each benchmarks (default `100`) * `NUM_CONTAINERS` to change the number of created containers for some benchmarks (e.g., `ps`) (default `100`) * `IMAGE` to change the default container image (default `docker.io/library/alpine:latest`) Signed-off-by: Valentin Rothberg --- hack/perf/README.md | 12 ++++++++++++ hack/perf/create.sh | 8 ++++++++ hack/perf/helpers.bash | 36 ++++++++++++++++++++++++++++++++++++ hack/perf/ps.sh | 9 +++++++++ hack/perf/rm.sh | 10 ++++++++++ hack/perf/run.sh | 18 ++++++++++++++++++ hack/perf/start.sh | 10 ++++++++++ hack/perf/stop.sh | 10 ++++++++++ 8 files changed, 113 insertions(+) create mode 100644 hack/perf/README.md create mode 100755 hack/perf/create.sh create mode 100644 hack/perf/helpers.bash create mode 100755 hack/perf/ps.sh create mode 100755 hack/perf/rm.sh create mode 100755 hack/perf/run.sh create mode 100755 hack/perf/start.sh create mode 100755 hack/perf/stop.sh diff --git a/hack/perf/README.md b/hack/perf/README.md new file mode 100644 index 0000000000..c806136212 --- /dev/null +++ b/hack/perf/README.md @@ -0,0 +1,12 @@ +# A set of scripts to compare the performance of two container engines + +Run the scripts via `sudo sh $script.sh`. + +WARNING: Running any script will run `systemd prune`. + +Use the following environment variables to change the default behavior: +* `ENGINE_A` to set container engine A (default `podman`) +* `ENGINE_B` to set container engine B (default `docker`) +* `RUNS` to change the runs/repetitions of each benchmarks (default `100`) +* `NUM_CONTAINERS` to change the number of created containers for some benchmarks (e.g., `ps`) (default `100`) +* `IMAGE` to change the default container image (default `docker.io/library/alpine:latest`) diff --git a/hack/perf/create.sh b/hack/perf/create.sh new file mode 100755 index 0000000000..45bf2b38ca --- /dev/null +++ b/hack/perf/create.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh +source ./helpers.bash + +setup +echo_bold "Create $RUNS containers" +hyperfine --warmup 10 --runs $RUNS \ + "$ENGINE_A create $IMAGE" \ + "$ENGINE_B create $IMAGE" diff --git a/hack/perf/helpers.bash b/hack/perf/helpers.bash new file mode 100644 index 0000000000..bfb576ffd4 --- /dev/null +++ b/hack/perf/helpers.bash @@ -0,0 +1,36 @@ +ENGINE_A=${ENGINE_A:-podman} +ENGINE_B=${ENGINE_B:-docker} +RUNS=${RUNS:-100} +NUM_CONTAINERS=${NUM_CONTAINERS:-100} +IMAGE=${IMAGE:-docker.io/library/alpine:latest} + +BOLD="$(tput bold)" +RESET="$(tput sgr0)" + +function echo_bold() { + echo "${BOLD}$1${RESET}" +} + +function pull_image() { + echo_bold "... pulling $IMAGE" + $ENGINE_A pull $IMAGE -q > /dev/null + $ENGINE_B pull $IMAGE -q > /dev/null +} + +function setup() { + echo_bold "---------------------------------------------------" + echo_bold "... comparing $ENGINE_A with $ENGINE_B" + echo_bold "... cleaning up previous containers and images" + $ENGINE_A system prune -f > /dev/null + $ENGINE_B system prune -f > /dev/null + pull_image + echo "" +} + +function create_containers() { + echo_bold "... creating $NUM_CONTAINERS containers" + for i in $(eval echo "{0..$NUM_CONTAINERS}"); do + $ENGINE_A create $IMAGE >> /dev/null + $ENGINE_B create $IMAGE >> /dev/null + done +} diff --git a/hack/perf/ps.sh b/hack/perf/ps.sh new file mode 100755 index 0000000000..7fe5441df2 --- /dev/null +++ b/hack/perf/ps.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh +source ./helpers.bash + +setup +echo_bold "List $NUM_CONTAINERS created containers" +create_containers +hyperfine --warmup 10 --runs $RUNS \ + "$ENGINE_A ps -a" \ + "$ENGINE_B ps -a" diff --git a/hack/perf/rm.sh b/hack/perf/rm.sh new file mode 100755 index 0000000000..0a38322c11 --- /dev/null +++ b/hack/perf/rm.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +source ./helpers.bash + +setup +echo_bold "Remove $RUNS containers in a row" +hyperfine --warmup 10 --runs $RUNS \ + --prepare "$ENGINE_A create --name=123 $IMAGE" \ + --prepare "$ENGINE_B create --name=123 $IMAGE" \ + "$ENGINE_A rm 123" \ + "$ENGINE_B rm 123" diff --git a/hack/perf/run.sh b/hack/perf/run.sh new file mode 100755 index 0000000000..a1be714960 --- /dev/null +++ b/hack/perf/run.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env sh +source ./helpers.bash + +setup +echo_bold "Run $RUNS containers in a row" +hyperfine --warmup 10 --runs $RUNS \ + --prepare "$ENGINE_A rm -f 123 || true" \ + --prepare "$ENGINE_B rm -f 123 || true" \ + "$ENGINE_A run --name=123 $IMAGE true" \ + "$ENGINE_B run --name=123 $IMAGE true" + +setup +echo_bold "Run and remove $RUNS containers in a row" +hyperfine --warmup 10 --runs $RUNS \ + --prepare "$ENGINE_A rm -f 123 || true" \ + --prepare "$ENGINE_B rm -f 123 || true" \ + "$ENGINE_A run --rm --name=123 $IMAGE true" \ + "$ENGINE_B run --rm --name=123 $IMAGE true" diff --git a/hack/perf/start.sh b/hack/perf/start.sh new file mode 100755 index 0000000000..a5668d7f0e --- /dev/null +++ b/hack/perf/start.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +source ./helpers.bash + +setup +echo_bold "Start $RUNS containers in a row" +hyperfine --warmup 10 --runs $RUNS \ + --prepare "$ENGINE_A rm -f 123 || true; $ENGINE_A create --name=123 $IMAGE true" \ + --prepare "$ENGINE_B rm -f 123 || true; $ENGINE_B create --name=123 $IMAGE true" \ + "$ENGINE_A start 123" \ + "$ENGINE_B start 123" diff --git a/hack/perf/stop.sh b/hack/perf/stop.sh new file mode 100755 index 0000000000..f321fe4291 --- /dev/null +++ b/hack/perf/stop.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +source ./helpers.bash + +setup +echo_bold "Stop $RUNS containers in a row" +hyperfine --warmup 10 --runs $RUNS \ + --prepare "$ENGINE_A rm -f 123 || true; $ENGINE_A run -d --name=123 $IMAGE top" \ + --prepare "$ENGINE_B rm -f 123 || true; $ENGINE_B run -d --name=123 $IMAGE top" \ + "$ENGINE_A stop 123" \ + "$ENGINE_B stop 123"