diff --git a/.github/actions/check_cirrus_cron/cron_failures.sh b/.github/actions/check_cirrus_cron/cron_failures.sh index 0e669c19c1..f3a7203dac 100755 --- a/.github/actions/check_cirrus_cron/cron_failures.sh +++ b/.github/actions/check_cirrus_cron/cron_failures.sh @@ -14,7 +14,9 @@ elif [[ -z "$NAME_ID_FILEPATH" ]]; then # output filepath err $(printf "$_errfmt" "\$NAME_ID_FILEPATH") fi -mkdir -p artifacts +confirm_gha_environment + +mkdir -p ./artifacts cat > ./artifacts/query_raw.json << "EOF" query { ownerRepository(platform: "LINUX", owner: "@@OWNER@@", name: "@@REPO@@") { @@ -32,14 +34,11 @@ EOF # https://cirrus-ci.com/explorer owner=$(cut -d '/' -f 1 <<<"$GITHUB_REPOSITORY") repo=$(cut -d '/' -f 2 <<<"$GITHUB_REPOSITORY") -sed -i -r -e "s/@@OWNER@@/$owner/g" -e "s/@@REPO@@/$repo/g" ./artifacts/query_raw.json - -# Easier to debug in error-reply when query is compacted -tr -d '\n' < ./artifacts/query_raw.json | tr -s ' ' | tee ./artifacts/query.json | \ - jq --indent 4 --color-output . +sed -r -e "s/@@OWNER@@/$owner/g" -e "s/@@REPO@@/$repo/g" \ + ./artifacts/query_raw.json > ./artifacts/query.json if grep -q '@@' ./artifacts/query.json; then - err "Found unreplaced substitution token in raw query JSON" + err "Found unreplaced substitution token in query JSON" fi # The query should never ever return an empty-list, unless there are no cirrus-cron @@ -47,7 +46,7 @@ fi # be running anyway. filt_head='.data.ownerRepository.cronSettings' -gql $(./artifacts/query.json) "$filt_head" > ./artifacts/reply.json +gql "$(<./artifacts/query.json)" "$filt_head" > ./artifacts/reply.json # e.x. reply.json # { # "data": { @@ -87,5 +86,6 @@ records=$(wc --words "$NAME_ID_FILEPATH" | cut -d ' ' -f 1) failures=$((records/2)) # Set the output of this step. # Ref: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter +# shellcheck disable=SC2154 echo "failures::$failures" >> $GITHUB_OUTPUT echo "Total failed Cirrus-CI cron builds: $failures" diff --git a/.github/actions/check_cirrus_cron/lib.sh b/.github/actions/check_cirrus_cron/lib.sh index 70f08f8099..7e8d42e83b 100644 --- a/.github/actions/check_cirrus_cron/lib.sh +++ b/.github/actions/check_cirrus_cron/lib.sh @@ -8,10 +8,28 @@ msg() { # Must be called from top-level of script, not another function. err() { # Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions - msg "::error file=${BASH_SOURCE[1]},line=${BASH_LINENO[0]}::$@" + msg "::error file=${BASH_SOURCE[1]},line=${BASH_LINENO[0]}::$*" exit 1 } +confirm_gha_environment() { + local _err_fmt + _err_fmt="I don't seem to be running from a github-actions workflow" + # These are all defined by github-actions + # shellcheck disable=SC2154 + if [[ -z "$GITHUB_OUTPUT" ]]; then + err "$_err_fmt, \$GITHUB_OUTPUT is empty" + elif [[ -z "$GITHUB_WORKFLOW" ]]; then + err "$_err_fmt, \$GITHUB_WORKFLOW is empty" + elif [[ ! -d "$GITHUB_WORKSPACE" ]]; then + # Defined by github-actions + # shellcheck disable=SC2154 + err "$_err_fmt, \$GITHUB_WORKSPACE='$GITHUB_WORKSPACE' isn't a directory" + fi + + cd "$GITHUB_WORKSPACE" || false +} + # Using python3 here is a compromise for readability and # properly handling quote, control and unicode character encoding. escape_query() { @@ -45,6 +63,8 @@ gql() { msg "::error file=${BASH_SOURCE[1]},line=${BASH_LINENO[0]}::Invalid query JSON: $query" return 1 fi + # SECRET_CIRRUS_API_KEY is defined github secret + # shellcheck disable=SC2154 if output=$(curl \ --request POST \ --silent \ diff --git a/.github/actions/check_cirrus_cron/make_email_body.sh b/.github/actions/check_cirrus_cron/make_email_body.sh index ab5a717eb0..af38566286 100755 --- a/.github/actions/check_cirrus_cron/make_email_body.sh +++ b/.github/actions/check_cirrus_cron/make_email_body.sh @@ -9,14 +9,16 @@ set -eo pipefail source $(dirname "${BASH_SOURCE[0]}")/lib.sh _errfmt="Expecting %s value to not be empty" +# NAME_ID_FILEPATH is defined by workflow YAML +# shellcheck disable=SC2154 if [[ -z "$GITHUB_REPOSITORY" ]]; then err $(printf "$_errfmt" "\$GITHUB_REPOSITORY") -elif [[ -z "$GITHUB_WORKFLOW" ]]; then - err $(printf "$_errfmt" "\$GITHUB_WORKFLOW") elif [[ ! -r "$NAME_ID_FILEPATH" ]]; then err "Expecting \$NAME_ID_FILEPATH value ($NAME_ID_FILEPATH) to be a readable file" fi +confirm_gha_environment + mkdir -p artifacts ( echo "Detected one or more Cirrus-CI cron-triggered jobs have failed recently:" @@ -27,6 +29,8 @@ mkdir -p artifacts done < "$NAME_ID_FILEPATH" echo "" + # Defined by github-actions + # shellcheck disable=SC2154 echo "# Source: ${GITHUB_WORKFLOW} workflow on ${GITHUB_REPOSITORY}." # Separate content from sendgrid.com automatic footer. echo "" diff --git a/.github/actions/check_cirrus_cron/rerun_failed_tasks.sh b/.github/actions/check_cirrus_cron/rerun_failed_tasks.sh index 8432815d03..4dd118e578 100755 --- a/.github/actions/check_cirrus_cron/rerun_failed_tasks.sh +++ b/.github/actions/check_cirrus_cron/rerun_failed_tasks.sh @@ -25,12 +25,16 @@ set -eo pipefail source $(dirname "${BASH_SOURCE[0]}")/lib.sh _errfmt="Expecting %s value to not be empty" +# NAME_ID_FILEPATH is defined by workflow YAML +# shellcheck disable=SC2154 if [[ -z "$SECRET_CIRRUS_API_KEY" ]]; then err $(printf "$_errfmt" "\$SECRET_CIRRUS_API_KEY") elif [[ ! -r "$NAME_ID_FILEPATH" ]]; then # output from cron_failures.sh err $(printf "Expecting %s value to be a readable file" "\$NAME_ID_FILEPATH") fi +confirm_gha_environment + mkdir -p artifacts # If there are no tasks, don't fail reading the file truncate -s 0 ./artifacts/rerun_tids.txt @@ -92,7 +96,9 @@ cat "$NAME_ID_FILEPATH" | \ # Check-value returned if the gql call was successful canary=$(uuidgen) # Ensure the trailing ',' is stripped from the end (would be invalid JSON) - task_ids=$(printf '[%s]' $(printf '"%s",' ${rerun_tasks[@]} | head -c -1)) + # Rely on shell word-splitting in this case. + # shellcheck disable=SC2048 + task_ids=$(printf '[%s]' $(printf '"%s",' ${rerun_tasks[*]} | head -c -1)) rerun_m=" mutation { batchReRun(input: { @@ -105,8 +111,12 @@ cat "$NAME_ID_FILEPATH" | \ } " filter='.data.batchReRun.clientMutationId' - result=$(gql "$rerun_m" "$filter") - if [[ $(jq -r -e "$filter"<<<"$result") != "$canary" ]]; then - err "Attempt to re-run tasks for build $BID failed: ${rerun_tasks[@]}" + if [[ ! "$NAME" =~ "testing" ]]; then # see test.sh + result=$(gql "$rerun_m" "$filter") + if [[ $(jq -r -e "$filter"<<<"$result") != "$canary" ]]; then + err "Attempt to re-run tasks for build $BID failed: ${rerun_tasks[*]}" + fi + else + warn "Test-mode: Would have sent GraphQL request: '$rerun_m'" fi done diff --git a/.github/actions/check_cirrus_cron/test.sh b/.github/actions/check_cirrus_cron/test.sh new file mode 100644 index 0000000000..200aa11ec0 --- /dev/null +++ b/.github/actions/check_cirrus_cron/test.sh @@ -0,0 +1,89 @@ + + +# This script attempts to confirm functional github action scripts. +# It expects to be called from Cirrus-CI, in a special execution +# enironment. Any use outside this environment will probably fail. + +set -eo pipefail + +# Defined by setup_environment.sh +# shellcheck disable=SC2154 +if ! ((PREBUILD)); then + echo "Not operating under expected environment" + exit 1 +fi + +expect_regex() { + local expected_regex + local input_file + expected_regex="$1" + input_file="$2" + egrep -q "$expected_regex" $input_file || \ + die "No match to '$expected_regex' in '$(<$input_file)'" +} + +req_env_vars CIRRUS_CI CIRRUS_REPO_FULL_NAME CIRRUS_WORKING_DIR CIRRUS_BUILD_ID + +# Defined by the CI system +# shellcheck disable=SC2154 +cd $CIRRUS_WORKING_DIR || fail + +header="Testing cirrus-cron github-action script:" +msg "$header cron_failures.sh" + +base=$CIRRUS_WORKING_DIR/.github/actions/check_cirrus_cron +# Don't care about mktemp return value +# shellcheck disable=SC2155 +export GITHUB_OUTPUT=$(mktemp -p '' cron_failures_output_XXXX) +# CIRRUS_REPO_FULL_NAME checked above in req_env_vars +# shellcheck disable=SC2154 +export GITHUB_REPOSITORY="$CIRRUS_REPO_FULL_NAME" +# shellcheck disable=SC2155 +export GITHUB_WORKSPACE=$(mktemp -d -p '' cron_failures_workspace_XXXX) +export GITHUB_WORKFLOW="testing" +# shellcheck disable=SC2155 +export NAME_ID_FILEPATH=$(mktemp -p '' cron_failures_data_XXXX) +trap "rm -rf $GITHUB_OUTPUT $GITHUB_WORKSPACE $NAME_ID_FILEPATH" EXIT + +##### + +cd /tmp || fail +# Replace newlines and indentation to make egrep easier +if ! $base/cron_failures.sh |& \ + tr -s '[:space:]' ' ' > $GITHUB_WORKSPACE/output; then + die "Failed: $base/cron_failures.sh with output '$(<$GITHUB_WORKSPACE/output)'" +fi + +expect_regex \ + 'result.+data.+ownerRepository.+cronSettings.+endgroup' \ + "$GITHUB_WORKSPACE/output" + +##### + +msg "$header make_email_body.sh" +# It's possible no cirrus-cron jobs actually failed +echo '' >> "$NAME_ID_FILEPATH" +# Don't need to test stdout/stderr of this +if ! $base/make_email_body.sh; then + die "make_email_body.sh failed" +fi + +expect_regex \ + '^Detected.+Cirrus-CI.+failed.*' \ + "$GITHUB_WORKSPACE/artifacts/email_body.txt" + +##### + +msg "$header rerun_failed_tasks.sh" +export SECRET_CIRRUS_API_KEY=testing-nottherightkey +# test.sh is sensitive to the 'testing' name. Var. defined by cirrus-ci +# shellcheck disable=SC2154 +echo "testing $CIRRUS_BUILD_ID" > "$NAME_ID_FILEPATH" +if ! $base/rerun_failed_tasks.sh |& \ + tr -s '[:space:]' ' ' > $GITHUB_WORKSPACE/rerun_output; then + die "rerun_failed_tasks.sh failed" +fi + +expect_regex \ + "Posting GraphQL Query.+$CIRRUS_BUILD_ID.+Selecting.+re-run" \ + "$GITHUB_WORKSPACE/rerun_output" diff --git a/.github/workflows/check_cirrus_cron.yml b/.github/workflows/check_cirrus_cron.yml index 1162ba2e76..c38574e366 100644 --- a/.github/workflows/check_cirrus_cron.yml +++ b/.github/workflows/check_cirrus_cron.yml @@ -32,7 +32,7 @@ jobs: cron_failures: runs-on: ubuntu-latest steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + - uses: actions/checkout@v3 with: persist-credentials: false @@ -47,7 +47,7 @@ jobs: - if: steps.cron.outputs.failures > 0 name: Send failure notification e-mail # Ref: https://github.com/dawidd6/action-send-mail - uses: dawidd6/action-send-mail@a80d851dc950256421f1d1d735a2dc1ef314ac8f # v2.2.2 + uses: dawidd6/action-send-mail@v3.7.1 with: server_address: ${{secrets.ACTION_MAIL_SERVER}} server_port: 465 @@ -59,14 +59,14 @@ jobs: body: file://./artifacts/email_body.txt - if: always() - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 + uses: actions/upload-artifact@v3 with: name: ${{ github.job }}_artifacts path: artifacts/* - if: failure() name: Send error notification e-mail - uses: dawidd6/action-send-mail@a80d851dc950256421f1d1d735a2dc1ef314ac8f # v2.2.2 + uses: dawidd6/action-send-mail@3.7.1 with: server_address: ${{secrets.ACTION_MAIL_SERVER}} server_port: 465 diff --git a/.github/workflows/rerun_cirrus_cron.yml b/.github/workflows/rerun_cirrus_cron.yml index 7fd01b071d..7973e418ec 100644 --- a/.github/workflows/rerun_cirrus_cron.yml +++ b/.github/workflows/rerun_cirrus_cron.yml @@ -14,6 +14,7 @@ on: # Debug: Allow triggering job manually in github-actions WebUI workflow_dispatch: {} + env: # Debug-mode can reveal secrets, only enable by a secret value. # Ref: https://help.github.com/en/actions/configuring-and-managing-workflows/managing-a-workflow-run#enabling-step-debug-logging @@ -28,10 +29,10 @@ permissions: contents: read jobs: - cron_failures: + cron_rerun: runs-on: ubuntu-latest steps: - - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - uses: actions/checkout@v3 with: persist-credentials: false @@ -42,14 +43,15 @@ jobs: - if: steps.cron.outputs.failures > 0 shell: bash run: './.github/actions/check_cirrus_cron/rerun_failed_tasks.sh' - uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2 + + - uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2 with: name: ${{ github.job }}_artifacts path: artifacts/* - if: failure() name: Send error notification e-mail - uses: dawidd6/action-send-mail@a80d851dc950256421f1d1d735a2dc1ef314ac8f # v2.2.2 + uses: dawidd6/action-send-mail@v3.7.1 with: server_address: ${{secrets.ACTION_MAIL_SERVER}} server_port: 465 diff --git a/contrib/cirrus/prebuild.sh b/contrib/cirrus/prebuild.sh index ea05d90dcc..0fb9004ff0 100755 --- a/contrib/cirrus/prebuild.sh +++ b/contrib/cirrus/prebuild.sh @@ -9,8 +9,10 @@ set -eo pipefail # prevent wasting time on tests that can't succeed due to some # outage, failure, or missed expectation. +set -a source /etc/automation_environment source $AUTOMATION_LIB_PATH/common_lib.sh +set +a req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \ PODBIN_NAME PRIV_NAME DISTRO_NV AUTOMATION_LIB_PATH \ @@ -21,6 +23,7 @@ req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \ # shellcheck disable=SC2154 cd $CIRRUS_WORKING_DIR +msg "Checking Cirrus YAML" # Defined by CI config. # shellcheck disable=SC2154 showrun $SCRIPT_BASE/cirrus_yaml_test.py @@ -28,13 +31,19 @@ showrun $SCRIPT_BASE/cirrus_yaml_test.py # Defined by CI config. # shellcheck disable=SC2154 if [[ "${DISTRO_NV}" =~ fedora ]]; then + msg "Checking shell scripts" showrun ooe.sh dnf install -y ShellCheck # small/quick addition showrun shellcheck --color=always --format=tty \ --shell=bash --external-sources \ --enable add-default-case,avoid-nullary-conditions,check-unassigned-uppercase \ --exclude SC2046,SC2034,SC2090,SC2064 \ --wiki-link-count=0 --severity=warning \ - $SCRIPT_BASE/*.sh hack/get_ci_vm.sh + $SCRIPT_BASE/*.sh \ + ./.github/actions/check_cirrus_cron/* \ + hack/get_ci_vm.sh + + export PREBUILD=1 + showrun bash ${CIRRUS_WORKING_DIR}/.github/actions/check_cirrus_cron/test.sh fi msg "Checking 3rd party network service connectivity"