Merge pull request #16449 from cevich/localbenchmarks

Cirrus: Collect benchmarks on machine instances

.cirrus.yml

@@ -666,7 +666,10 @@ podman_machine_task:
     clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
-    always: *int_logs_artifacts
+    always: &machine_logs_benchmarks
+      <<: *int_logs_artifacts
+      benchmark_artifacts:
+          path: ./data/*
 
 
 podman_machine_aarch64_task:
@@ -692,7 +695,7 @@ podman_machine_aarch64_task:
     clone_script: *get_gosrc_aarch64
     setup_script: *setup
     main_script: *main
-    always: *int_logs_artifacts
+    always: *machine_logs_benchmarks
 
 
 # Always run subsequent to integration tests.  While parallelism is lost
@@ -1089,6 +1092,12 @@ artifacts_task:
         - tar xjf repo.tbz
         - mv ./podman-remote-release-darwin_*.zip $CIRRUS_WORKING_DIR/
         - mv ./contrib/pkginstaller/out/podman-installer-macos-*.pkg $CIRRUS_WORKING_DIR/
+    benchmarks_script:
+        - mkdir -p /tmp/benchmarks
+        - cd /tmp/benchmarks
+        - $ARTCURL/podman_machine/benchmark.zip
+        - unzip benchmark.zip
+        - mv ./data/benchmarks.{env,csv} $CIRRUS_WORKING_DIR/
     always:
       contents_script: ls -la $CIRRUS_WORKING_DIR
       # Produce downloadable files and an automatic zip-file accessible

Makefile

@@ -578,11 +578,11 @@ localintegration: test-binaries ginkgo
 remoteintegration: test-binaries ginkgo-remote
 
 .PHONY: localmachine
-localmachine: test-binaries
+localmachine: test-binaries .install.ginkgo
 	$(MAKE) ginkgo-run GINKGONODES=1 GINKGOWHAT=pkg/machine/e2e/. HACK=
 
 .PHONY: localbenchmarks
-localbenchmarks: test-binaries
+localbenchmarks: install.tools test-binaries
 	PATH=$(PATH):$(shell pwd)/hack ACK_GINKGO_RC=true $(GINKGO) \
 		      -focus "Podman Benchmark Suite" \
 		      -tags "$(BUILDTAGS) benchmarks" -noColor \

contrib/cirrus/lib.sh

@@ -93,7 +93,7 @@ EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA"
 # contexts, such as host->container or root->rootless user
 #
 # List of envariables which must be EXACT matches
-PASSTHROUGH_ENV_EXACT='CGROUP_MANAGER|DEST_BRANCH|DISTRO_NV|GOCACHE|GOPATH|GOSRC|NETWORK_BACKEND|OCI_RUNTIME|ROOTLESS_USER|SCRIPT_BASE|SKIP_USERNS'
+PASSTHROUGH_ENV_EXACT='CGROUP_MANAGER|DEST_BRANCH|DISTRO_NV|GOCACHE|GOPATH|GOSRC|NETWORK_BACKEND|OCI_RUNTIME|ROOTLESS_USER|SCRIPT_BASE|SKIP_USERNS|EC2_INST_TYPE'
 
 # List of envariable patterns which must match AT THE BEGINNING of the name.
 PASSTHROUGH_ENV_ATSTART='CI|TEST'
@@ -290,3 +290,32 @@ remove_packaged_podman_files() {
     # Be super extra sure and careful vs performant and completely safe
     sync && echo 3 > /proc/sys/vm/drop_caches || true
 }
+
+# Execute make localbenchmarks in $CIRRUS_WORKING_DIR/data
+# for preserving as a task artifact.
+localbenchmarks() {
+    local datadir
+    req_env_vars DISTRO_NV PODBIN_NAME PRIV_NAME TEST_ENVIRON TEST_FLAVOR
+    req_env_vars VM_IMAGE_NAME EC2_INST_TYPE
+
+    datadir=$CIRRUS_WORKING_DIR/data
+    mkdir -p $datadir
+
+    (
+      echo "# Env. var basis for benchmarks benchmarks."
+      printenv | grep -Ev "$SECRET_ENV_RE" | sort
+
+      echo "# Machine details for data-comparison sake, not actual env. vars."
+      # Checked above in req_env_vars
+      # shellcheck disable=SC2154
+      echo "\
+CPUTOTAL=$(grep -ce '^processor' /proc/cpuinfo)
+INST_TYPE=$EC2_INST_TYPE  # one day may include other cloud's VM types.
+MEMTOTAL=$(awk -F: '$1 == "MemTotal" { print $2 }' </proc/meminfo | sed -e "s/^ *//")
+UNAME_RM=$(uname -r -m)
+"
+    ) > $datadir/benchmarks.env
+    make localbenchmarks | tee $datadir/benchmarks.raw
+    msg "Processing raw benchmarks output"
+    hack/parse-localbenchmarks < $datadir/benchmarks.raw | tee $datadir/benchmarks.csv
+}

contrib/cirrus/prebuild.sh

@@ -33,7 +33,7 @@ showrun $SCRIPT_BASE/cirrus_yaml_test.py
 if [[ "${DISTRO_NV}" =~ fedora ]]; then
     msg "Checking shell scripts"
     showrun ooe.sh dnf install -y ShellCheck  # small/quick addition
-    showrun shellcheck --color=always --format=tty \
+    showrun shellcheck --format=tty \
         --shell=bash --external-sources \
         --enable add-default-case,avoid-nullary-conditions,check-unassigned-uppercase \
         --exclude SC2046,SC2034,SC2090,SC2064 \

contrib/cirrus/runner.sh

@@ -394,6 +394,9 @@ dotest() {
 }
 
 _run_machine() {
+    # This environment is convenient for executing some benchmarking
+    localbenchmarks
+
     # N/B: Can't use _bail_if_test_can_be_skipped here b/c content isn't under test/
     make localmachine |& logformatter
 }