From f95dd21d5d72ac85976471de7a41dca1758593a3 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Thu, 30 Nov 2023 06:29:33 -0700 Subject: [PATCH] deferred test failures: handle a corner case Followup to #20797 (defer assertion failures). The bail-now() helper was being defined only in setup() ... and some tests, particularly 001-basic.bats, define their own minimalist setup(). Symptom was "bail-now: command not found", which still caused test to fail (so no failures were hidden) but led to concern and wasted time when analyzing failures. Solution: add one more definition of bail-now(), in outer scope. There is still one pathological case I'm not addressing: a bats file that defines its own teardown() which does not invoke basic_teardown(), then has a test that runs defer-assertion-failures without a followup immediate-assertion-failures. This would lead to failures that are never seen. Since teardown() without basic_teardown() is invalid, I choose not to worry about this case. Signed-off-by: Ed Santiago --- test/system/505-networking-pasta.bats | 4 ---- test/system/helpers.bash | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/test/system/505-networking-pasta.bats b/test/system/505-networking-pasta.bats index af342ddc84..944ea6b4e3 100644 --- a/test/system/505-networking-pasta.bats +++ b/test/system/505-networking-pasta.bats @@ -254,10 +254,6 @@ function pasta_test_do() { assert "${output}" = "${expect}" "Mismatch between data sent and received" } -function teardown() { - rm -f "${XFER_FILE}" -} - ### Addresses ################################################################## @test "IPv4 default address assignment" { diff --git a/test/system/helpers.bash b/test/system/helpers.bash index d69b717205..ded6de6425 100644 --- a/test/system/helpers.bash +++ b/test/system/helpers.bash @@ -210,10 +210,20 @@ function basic_setup() { immediate-assertion-failures } +# bail-now is how we terminate a test upon assertion failure. +# By default, and the vast majority of the time, it just triggers +# immediate test termination; but see defer-assertion-failures, below. +function bail-now() { + # "false" does not apply to "bail now"! It means "nonzero exit", + # which BATS interprets as "yes, bail immediately". + false +} + +# Invoked on teardown: will terminate immediately if there have been +# any deferred test failures; otherwise will reset back to immediate +# test termination on future assertions. function immediate-assertion-failures() { function bail-now() { - # "false" does not apply to "bail now"! It means "nonzero exit", - # which BATS interprets as "yes, bail immediately". false } @@ -225,6 +235,9 @@ function immediate-assertion-failures() { fi } +# Used in special test circumstances--typically multi-condition loops--to +# continue going even on assertion failures. The test will fail afterward, +# usually in teardown. This can be useful to show failure patterns. function defer-assertion-failures() { function bail-now() { ASSERTION_FAILURES+="!"