minikube: instrument tests, to allow debugging failures

New run_minikube() helper, modeled after run_podman(). Echoes
each command being run and its output. On failure, runs minikube logs.

Addresses (does not close) #21931 which is hitting us hard in CI.
Probably quay flakes, but it's impossible to tell without logs.

Also: bug fix: one "run podman" fixed to run_podman

Signed-off-by: Ed Santiago <santiago@redhat.com>
This commit is contained in:
Ed Santiago
2024-03-19 10:02:43 -06:00
parent 8a643c243e
commit aa5c4879b4
2 changed files with 91 additions and 80 deletions

View File

@ -4,10 +4,55 @@ load ../system/helpers.bash
KUBECONFIG="$HOME/.kube/config"
##################
# run_minikube # Local helper, with instrumentation for debugging failures
##################
function run_minikube() {
# Number as first argument = expected exit code; default 0
local expected_rc=0
case "$1" in
[0-9]) expected_rc=$1; shift;;
[1-9][0-9]) expected_rc=$1; shift;;
[12][0-9][0-9]) expected_rc=$1; shift;;
'?') expected_rc= ; shift;; # ignore exit code
esac
# stdout is only emitted upon error; this printf is to help in debugging
printf "\n%s %s %s %s\n" "$(timestamp)" "\$" "minikube" "$*"
run minikube "$@"
# without "quotes", multiple lines are glommed together into one
if [[ -n "$output" ]]; then
echo "$(timestamp) $output"
fi
if [[ "$status" -ne 0 ]]; then
echo -n "$(timestamp) [ rc=$status ";
if [[ -n "$expected_rc" ]]; then
if [[ "$status" -eq "$expected_rc" ]]; then
echo -n "(expected) ";
else
echo -n "(** EXPECTED $expected_rc **) ";
fi
fi
echo "]"
fi
if [[ -n "$expected_rc" ]]; then
if [[ "$status" -ne "$expected_rc" ]]; then
# Further debugging
echo "\$ minikube logs"
run minikube logs
echo "$output"
die "exit code is $status; expected $expected_rc"
fi
fi
}
function setup(){
# only set up the minikube cluster before the first test
if [[ "$BATS_TEST_NUMBER" -eq 1 ]]; then
minikube start
run_minikube start
wait_for_default_sa
fi
basic_setup
@ -17,8 +62,12 @@ function teardown(){
# only delete the minikube cluster if we are done with the last test
# the $DEBUG_MINIKUBE env can be set to preserve the cluster to debug if needed
if [[ "$BATS_TEST_NUMBER" -eq ${#BATS_TEST_NAMES[@]} ]] && [[ "$DEBUG_MINIKUBE" == "" ]]; then
minikube delete
run_minikube delete
fi
# Prevents nasty red warnings in log
run_podman rmi --ignore $(pause_image)
basic_teardown
}
@ -29,8 +78,7 @@ function wait_for_default_sa(){
# if the default service account hasn't been created yet, there is something else wrong
while [[ $count -lt 30 ]] && [[ $sa_ready == false ]]
do
run minikube kubectl get sa
assert "$status" -eq 0
run_minikube kubectl get sa
if [[ "$output" != "No resources found in default namespace." ]]; then
sa_ready=true
fi
@ -49,7 +97,7 @@ function wait_for_pods_to_start(){
# if the pod hasn't started running after 30 seconds, there is something else wrong
while [[ $count -lt 30 ]] && [[ $running == false ]]
do
run minikube kubectl get pods
run_minikube kubectl get pods
assert "$status" -eq 0
if [[ "$output" =~ "Running" ]]; then
running=true