espressif/arduino-esp32
1
0
Fork 0
mirror of https://github.com/espressif/arduino-esp32.git synced 2026-03-13 09:41:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): Fix tests and workflows (#12024)

Browse Source 
* fix(ci): Fix tests and workflows

* feat(tests): Add job ID fixture

* fix(tests): Fix performance test result generation

* ci(pre-commit): Apply automatic fixes

---------

Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
This commit is contained in:
Lucas Saavedra Vaz
2025-11-20 07:10:21 -03:00
committed by GitHub
parent a90a523996
commit 5ab1db62ef
16 changed files with 147 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
.github/scripts/tests_run.sh vendored
View File

@@ -111,15 +111,23 @@ function run_test {
rm "$sketchdir"/diagram.json 2>/dev/null || true
local wifi_args=""
if [ -n "$wifi_ssid" ]; then
wifi_args="--wifi-ssid \"$wifi_ssid\""
fi
if [ -n "$wifi_password" ]; then
wifi_args="$wifi_args --wifi-password \"$wifi_password\""
fi
result=0
printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}" "$wifi_args"
bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q} $wifi_args; exit \$?" || result=$?
printf "\n"
if [ $result -ne 0 ]; then
result=0
printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i"
printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}" "$wifi_args"
bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q} $wifi_args; exit \$?" || result=$?
printf "\n"
if [ $result -ne 0 ]; then
printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i"
@@ -137,6 +145,8 @@ platform="hardware"
chunk_run=0
options=0
erase=0
wifi_ssid=""
wifi_password=""
while [ -n "$1" ]; do
case $1 in
@@ -188,6 +198,14 @@ while [ -n "$1" ]; do
shift
test_type=$1
;;
-wifi-ssid )
shift
wifi_ssid=$1
;;
-wifi-password )
shift
wifi_password=$1
;;
* )
break
;;

2
.github/workflows/release.yml vendored
View File

@@ -68,7 +68,7 @@ jobs:
mkdir -p ${{ github.workspace }}/hosted
# Copy hosted binaries to proper directory without overwriting existing files
cp -n ${{ github.workspace }}/hosted-latest/*.bin ${{ github.workspace }}/hosted/
cp --update=none ${{ github.workspace }}/hosted-latest/*.bin ${{ github.workspace }}/hosted/
# Commit the changes
git config user.name "github-actions[bot]"

14
.github/workflows/tests_hw_wokwi.yml vendored
View File

@@ -361,7 +361,14 @@ jobs:
download_artifacts: 'true'
download_artifacts_on_failure: 'true'
download_path: './gitlab-artifacts'
variables: '{"TEST_TYPES":"${{ steps.prepare-variables.outputs.test_types }}","TEST_CHIPS":"${{ steps.prepare-variables.outputs.test_chips }}","PIPELINE_ID":"${{ env.id }}","BINARIES_RUN_ID":"${{ github.event.workflow_run.id }}","GITHUB_REPOSITORY":"${{ github.repository }}"}'
variables: >-
{
"TEST_TYPES":"${{ steps.prepare-variables.outputs.test_types }}",
"TEST_CHIPS":"${{ steps.prepare-variables.outputs.test_chips }}",
"PIPELINE_ID":"${{ env.id }}",
"BINARIES_RUN_ID":"${{ github.event.workflow_run.id }}",
"GITHUB_REPOSITORY":"${{ github.repository }}"
}
- name: Process Downloaded Artifacts
if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }}
@@ -536,8 +543,11 @@ jobs:
if: ${{ steps.check-tests.outputs.enabled == 'true' }}
env:
WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}
WOKWI_WIFI_SSID: "Wokwi-GUEST"
# The Wokwi Wi-Fi does not have a password, so we use an empty string
WOKWI_WIFI_PASSWORD: ""
run: |
bash .github/scripts/tests_run.sh -c -type ${{ matrix.type }} -t ${{ matrix.chip }} -i 0 -m 1 -W
bash .github/scripts/tests_run.sh -c -type "${{ matrix.type }}" -t "${{ matrix.chip }}" -i 0 -m 1 -W -wifi-ssid "${{ env.WOKWI_WIFI_SSID }}" -wifi-password "${{ env.WOKWI_WIFI_PASSWORD }}"
- name: Upload ${{ matrix.chip }} ${{ matrix.type }} Wokwi results as cache
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3

2
.gitlab/workflows/hw_test_template.yml
View File

@@ -50,7 +50,7 @@ hw-test-template:
[ -z "$d" ] && continue;
sketch=$(basename "$d");
echo Running $sketch in $d;
bash .github/scripts/tests_run.sh -t $TEST_CHIP -s $sketch -e || rc=$?;
bash .github/scripts/tests_run.sh -t "$TEST_CHIP" -s "$sketch" -e -wifi-ssid "$RUNNER_WIFI_SSID" -wifi-password "$RUNNER_WIFI_PASSWORD" || rc=$?;
done <<< "$TEST_LIST"; exit $rc
artifacts:

22
tests/conftest.py Normal file
View File

@@ -0,0 +1,22 @@
import pytest
import os
def pytest_addoption(parser):
parser.addoption("--wifi-password", action="store", default=None, help="Wi-Fi password.")
parser.addoption("--wifi-ssid", action="store", default=None, help="Wi-Fi SSID.")
@pytest.fixture(scope="session")
def wifi_ssid(request):
return request.config.getoption("--wifi-ssid")
@pytest.fixture(scope="session")
def wifi_pass(request):
return request.config.getoption("--wifi-password")
@pytest.fixture(scope="session")
def ci_job_id(request):
return os.environ.get("CI_JOB_ID")

3
tests/performance/coremark/test_coremark.py
View File

@@ -45,13 +45,14 @@ def test_coremark(dut, request):
results = {"coremark": {"runs": runs, "cores": cores, "avg_score": avg_score}}
current_folder = os.path.dirname(request.path)
os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
file_index = 0
report_file = os.path.join(current_folder, dut.app.target, "result_coremark" + str(file_index) + ".json")
while os.path.exists(report_file):
report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
file_index += 1
with open(report_file, "w") as f:
with open(report_file, "w+") as f:
try:
f.write(json.dumps(results))
except Exception as e:

3
tests/performance/fibonacci/test_fibonacci.py
View File

@@ -67,13 +67,14 @@ def test_fibonacci(dut, request):
results = {"fibonacci": {"runs": runs, "fib_n": fib_n, "avg_time": avg_time}}
current_folder = os.path.dirname(request.path)
os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
file_index = 0
report_file = os.path.join(current_folder, dut.app.target, "result_fibonacci" + str(file_index) + ".json")
while os.path.exists(report_file):
report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
file_index += 1
with open(report_file, "w") as f:
with open(report_file, "w+") as f:
try:
f.write(json.dumps(results))
except Exception as e:

3
tests/performance/linpack_double/test_linpack_double.py
View File

@@ -48,13 +48,14 @@ def test_linpack_double(dut, request):
results = {"linpack_double": {"runs": runs, "avg_score": avg_score, "min_score": min_score, "max_score": max_score}}
current_folder = os.path.dirname(request.path)
os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
file_index = 0
report_file = os.path.join(current_folder, dut.app.target, "result_linpack_double" + str(file_index) + ".json")
while os.path.exists(report_file):
report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
file_index += 1
with open(report_file, "w") as f:
with open(report_file, "w+") as f:
try:
f.write(json.dumps(results))
except Exception as e:

3
tests/performance/linpack_float/test_linpack_float.py
View File

@@ -48,13 +48,14 @@ def test_linpack_float(dut, request):
results = {"linpack_float": {"runs": runs, "avg_score": avg_score, "min_score": min_score, "max_score": max_score}}
current_folder = os.path.dirname(request.path)
os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
file_index = 0
report_file = os.path.join(current_folder, dut.app.target, "result_linpack_float" + str(file_index) + ".json")
while os.path.exists(report_file):
report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
file_index += 1
with open(report_file, "w") as f:
with open(report_file, "w+") as f:
try:
f.write(json.dumps(results))
except Exception as e:

3
tests/performance/psramspeed/test_psramspeed.py
View File

@@ -92,13 +92,14 @@ def test_psramspeed(dut, request):
results = {"psramspeed": {"runs": runs, "copies": copies, "max_test_size": max_test_size, "results": avg_results}}
current_folder = os.path.dirname(request.path)
os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
file_index = 0
report_file = os.path.join(current_folder, dut.app.target, "result_psramspeed" + str(file_index) + ".json")
while os.path.exists(report_file):
report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
file_index += 1
with open(report_file, "w") as f:
with open(report_file, "w+") as f:
try:
f.write(json.dumps(results))
except Exception as e:

4
tests/performance/ramspeed/ramspeed.ino
View File

@@ -234,8 +234,8 @@ void setup() {
delay(10);
}
void *dest = malloc(MAX_TEST_SIZE);
const void *src = malloc(MAX_TEST_SIZE);
void *dest = heap_caps_malloc(MAX_TEST_SIZE, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL);
const void *src = heap_caps_malloc(MAX_TEST_SIZE, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL);
if (!dest || !src) {
Serial.println("Memory allocation failed");

3
tests/performance/ramspeed/test_ramspeed.py
View File

@@ -92,13 +92,14 @@ def test_ramspeed(dut, request):
results = {"ramspeed": {"runs": runs, "copies": copies, "max_test_size": max_test_size, "results": avg_results}}
current_folder = os.path.dirname(request.path)
os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
file_index = 0
report_file = os.path.join(current_folder, dut.app.target, "result_ramspeed" + str(file_index) + ".json")
while os.path.exists(report_file):
report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
file_index += 1
with open(report_file, "w") as f:
with open(report_file, "w+") as f:
try:
f.write(json.dumps(results))
except Exception as e:

3
tests/performance/superpi/test_superpi.py
View File

@@ -40,13 +40,14 @@ def test_superpi(dut, request):
results = {"superpi": {"runs": runs, "digits": digits, "avg_time": avg_time}}
current_folder = os.path.dirname(request.path)
os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
file_index = 0
report_file = os.path.join(current_folder, dut.app.target, "result_superpi" + str(file_index) + ".json")
while os.path.exists(report_file):
report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
file_index += 1
with open(report_file, "w") as f:
with open(report_file, "w+") as f:
try:
f.write(json.dumps(results))
except Exception as e:

3
tests/pytest.ini
View File

@@ -1,5 +1,8 @@
[pytest]
addopts = --embedded-services esp,arduino,wokwi,qemu
junit_family = xunit1
filterwarnings =
ignore::pytest.PytestExperimentalApiWarning
# log related
log_cli = True

27
tests/validation/wifi/test_wifi.py
View File

@@ -1,13 +1,36 @@
import logging
import pytest
def test_wifi(dut):
def test_wifi(dut, wifi_ssid, wifi_pass):
LOGGER = logging.getLogger(__name__)
# Fail if no WiFi SSID is provided
if not wifi_ssid:
pytest.fail("WiFi SSID is required but not provided. Use --wifi-ssid argument.")
# Wait for device to be ready and send WiFi credentials
LOGGER.info("Waiting for device to be ready...")
dut.expect_exact("Device ready for WiFi credentials")
dut.expect_exact("Send SSID:")
LOGGER.info(f"Sending WiFi credentials: SSID={wifi_ssid}")
dut.write(f"{wifi_ssid}")
dut.expect_exact("Send Password:")
LOGGER.info(f"Sending WiFi password: Password={wifi_pass}")
dut.write(f"{wifi_pass or ''}")
# Verify credentials were received
dut.expect_exact(f"SSID: {wifi_ssid}")
dut.expect_exact(f"Password: {wifi_pass or ''}")
LOGGER.info("Starting WiFi Scan")
dut.expect_exact("Scan start")
dut.expect_exact("Scan done")
dut.expect_exact("Wokwi-GUEST")
LOGGER.info(f"Looking for WiFi network: {wifi_ssid}")
dut.expect_exact(wifi_ssid)
LOGGER.info("WiFi Scan done")
LOGGER.info("Connecting to WiFi")

48
tests/validation/wifi/wifi.ino
View File

@@ -38,8 +38,8 @@
#include <WiFi.h>
const char *ssid = "Wokwi-GUEST";
const char *password = "";
String ssid = "";
String password = "";
// WARNING: This function is called from a separate FreeRTOS task (thread)!
void WiFiEvent(WiFiEvent_t event) {
@@ -87,14 +87,56 @@ void WiFiGotIP(WiFiEvent_t event, WiFiEventInfo_t info) {
Serial.println(IPAddress(info.got_ip.ip_info.ip.addr));
}
void readWiFiCredentials() {
Serial.println("Waiting for WiFi credentials...");
Serial.println("Send SSID:");
// Wait for SSID
while (ssid.length() == 0) {
if (Serial.available()) {
ssid = Serial.readStringUntil('\n');
ssid.trim();
}
delay(100);
}
Serial.println("Send Password:");
// Wait for password (allow empty password)
bool password_received = false;
while (!password_received) {
if (Serial.available()) {
password = Serial.readStringUntil('\n');
password.trim();
password_received = true; // Accept even empty password
}
delay(100);
}
Serial.print("SSID: ");
Serial.println(ssid);
Serial.print("Password: ");
Serial.println(password);
}
void setup() {
Serial.begin(115200);
while (!Serial) {
delay(100);
}
// delete old config
WiFi.disconnect(true);
delay(1000);
// Wait for test to be ready
Serial.println("Device ready for WiFi credentials");
// Read WiFi credentials from serial
readWiFiCredentials();
// Examples of different ways to register wifi events;
// these handlers will be called from another thread.
WiFi.onEvent(WiFiEvent);
@@ -134,7 +176,7 @@ void setup() {
// Delete the scan result to free memory for code below.
WiFi.scanDelete();
WiFi.begin(ssid, password);
WiFi.begin(ssid.c_str(), password.c_str());
Serial.println();
Serial.println();