Skip to content

Commit 41a15b2

Browse files
PetrESPPetrESP
committed
Pytest suite rework
1 parent 5a4f985 commit 41a15b2

File tree

9 files changed

+82
-179
lines changed

9 files changed

+82
-179
lines changed

‎.github/workflows/build-run-applications.yml‎

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,7 @@ on:
1616
schedule:
1717
- cron: '0 0 * * *' # every day at midnight
1818
workflow_dispatch:
19-
inputs:
20-
WFType:
21-
description: 'Workflow type'
22-
required: true
23-
default: 'Build + Tests'
24-
type: choice
25-
options:
26-
- Build + Tests
27-
- Build + Tests + Benchmark
19+
2820

2921
# Cancel previous CI, if running and changed label or pushed PR (Prevent to wait for runners)
3022
concurrency:
@@ -142,17 +134,9 @@ jobs:
142134
matrix:
143135
idf_ver:
144136
- "release-v5.5"
145-
runner:
146-
- example: "test_example_display"
147-
- example: "test_example_sdcard"
148-
- example: "test_example_lvgl_demos"
149-
- example: "test_example_generic_button_led"
150-
- example: "test_example_lvgl_benchmark"
151137
env:
152138
TEST_RESULT_NAME: test_results_${{ matrix.runner.example }}_${{ matrix.idf_ver }}
153-
BENCHMARK_RESULT_NAME: benchmark_${{ matrix.runner.example }}_${{ matrix.idf_ver }}
154139
TEST_RESULT_FILE: test_results_${{ matrix.runner.example }}_${{ matrix.idf_ver }}.xml
155-
PYTEST_BENCHMARK_IGNORE: ${{ (contains(github.event.pull_request.labels.*.name, 'Run benchmark') || contains(inputs.WFType, 'Build + Tests + Benchmark') || github.ref_name == 'master') && format(' ') || format('--ignore=examples/display_lvgl_benchmark') }}
156140
runs-on: [self-hosted, Linux, bspwall]
157141
container:
158142
image: espressif/idf:${{ matrix.idf_ver }}
@@ -177,7 +161,7 @@ jobs:
177161
178162
export PYTEST_EMBEDDED_CACHE_DIR=/tmp/pytest-embedded-cache-dummy
179163
mkdir -p /tmp/pytest-embedded-cache-dummy
180-
pytest --suppress-no-test-exit-code --ignore-glob '*/managed_components/*' --ignore=.github --junit-xml=${{ env.TEST_RESULT_FILE }} -p no:idf-ci -k "${{ matrix.runner.example }} and not test_example_display_sensors" -n auto ${{ env.PYTEST_BENCHMARK_IGNORE }}
164+
pytest --junit-xml=${{ env.TEST_RESULT_FILE }} -n auto ${{ env.PYTEST_BENCHMARK_IGNORE }} --dist loadgroup
181165
- name: Upload test results
182166
uses: actions/upload-artifact@v4
183167
if: always()

‎README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,6 @@ Here is a summary of the available examples:
8282
| [Sensors Example](examples/display_sensors) | Acquire sensor data using the sensor hub component | <details><summary>1 board</summary>[esp-box-3](bsp/esp-box-3)</details> | [Flash Example](https://espressif.github.io/esp-launchpad/?flashConfigURL=https://espressif.github.io/esp-bsp/config.toml&app=display_sensors) |
8383
| [USB HID Example](examples/display_usb_hid) | USB HID demo (keyboard, mouse, or gamepad visualization using LVGL) | <details><summary>3 boards</summary>[esp32_p4_function_ev_board](bsp/esp32_p4_function_ev_board)<br/>[esp32_s3_usb_otg](bsp/esp32_s3_usb_otg)<br/>[m5stack_tab5](bsp/m5stack_tab5)</details> | - |
8484
| [Generic Button and LED Example](examples/generic_button_led) | Minimal example using the Generic BSP: button and LED control | <details><summary>2 boards</summary>[esp_bsp_devkit](bsp/esp_bsp_devkit)<br/>[esp_bsp_generic](bsp/esp_bsp_generic)</details> | - |
85-
| [MQTT Example](examples/mqtt_example) | Collect sensor data and publish to an MQTT server | <details><summary>1 board</summary>[esp32_azure_iot_kit](bsp/esp32_azure_iot_kit)</details> | - |
8685

8786
<!-- EXAMPLES_TABLE_END -->
8887

‎conftest.py‎

Lines changed: 16 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,113 +1,32 @@
1-
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
1+
# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD
22
# SPDX-License-Identifier: Apache-2.0
33
import os
44
import uuid
55
import pytest
66

77

8-
def pytest_generate_tests(metafunc):
9-
allowed_ids = set()
10-
11-
# get markers from test case (e.g. 'esp_box_3')
12-
for marker in metafunc.definition.iter_markers():
13-
allowed_ids.add(marker.name)
8+
# TODO: Rename esp-box-3 port in bsp-wall to esp_box_3
149

15-
all_params = [
16-
pytest.param('/dev/boards/esp-box-3',
17-
'/dev/boards/esp-box-3',
18-
'build_esp-box-3',
19-
id='esp_box_3'),
20-
pytest.param('/dev/boards/esp32_c3_lcdkit',
21-
'/dev/boards/esp32_c3_lcdkit',
22-
'build_esp32_c3_lcdkit',
23-
id='esp32_c3_lcdkit'),
24-
pytest.param('/dev/boards/esp32_p4_box',
25-
'/dev/boards/esp32_p4_box',
26-
'build_esp32_p4_box',
27-
id='esp32_p4_box'),
28-
pytest.param('/dev/boards/esp32_p4_function_ev_board',
29-
'/dev/boards/esp32_p4_function_ev_board',
30-
'build_esp32_p4_function_ev_board',
31-
id='esp32_p4_function_ev_board'),
32-
pytest.param('/dev/boards/esp32_s3_eye',
33-
'/dev/boards/esp32_s3_eye',
34-
'build_esp32_s3_eye',
35-
id='esp32_s3_eye'),
36-
pytest.param('/dev/boards/esp32_s3_lcd_ev_board',
37-
'/dev/boards/esp32_s3_lcd_ev_board',
38-
'build_esp32_s3_lcd_ev_board',
39-
id='esp32_s3_lcd_ev_board'),
40-
pytest.param('/dev/boards/esp32_s3_lcd_ev_board-2',
41-
'/dev/boards/esp32_s3_lcd_ev_board-2',
42-
'build_esp32_s3_lcd_ev_board',
43-
id='esp32_s3_lcd_ev_board_2'),
44-
pytest.param('/dev/boards/esp32_s3_usb_otg',
45-
'/dev/boards/esp32_s3_usb_otg',
46-
'build_esp32_s3_usb_otg',
47-
id='esp32_s3_usb_otg'),
48-
pytest.param('/dev/boards/esp_wrover_kit',
49-
'/dev/boards/esp_wrover_kit',
50-
'build_esp_wrover_kit',
51-
id='esp_wrover_kit'),
52-
pytest.param('/dev/boards/esp32_azure_iot_kit',
53-
'/dev/boards/esp32_azure_iot_kit',
54-
'build_esp32_azure_iot_kit',
55-
id='esp32_azure_iot_kit'),
56-
pytest.param('/dev/boards/m5dial',
57-
'/dev/boards/m5dial',
58-
'build_m5dial',
59-
id='m5dial'),
60-
pytest.param('/dev/boards/m5stack_core',
61-
'/dev/boards/m5stack_core',
62-
'build_m5stack_core',
63-
id='m5stack_core'),
64-
pytest.param('/dev/boards/m5stack_core_2',
65-
'/dev/boards/m5stack_core_2',
66-
'build_m5stack_core',
67-
id='m5stack_core_2'),
68-
pytest.param('/dev/boards/m5stack_core_s3',
69-
'/dev/boards/m5stack_core_s3',
70-
'build_m5stack_core_s3',
71-
id='m5stack_core_s3'),
72-
pytest.param('/dev/boards/m5stack_core_s3_se',
73-
'/dev/boards/m5stack_core_s3_se',
74-
'build_m5stack_core_s3',
75-
id='m5stack_core_s3_se'),
76-
pytest.param('/dev/boards/m5_atom_s3',
77-
'/dev/boards/m5_atom_s3',
78-
'build_m5_atom_s3',
79-
id='m5_atom_s3'),
80-
pytest.param('/dev/boards/esp32_s3_devkitc_1_1',
81-
'/dev/boards/esp32_s3_devkitc_1_1',
82-
'build_esp_bsp_devkit',
83-
id='esp_bsp_devkit'),
84-
pytest.param('/dev/boards/esp32_s2_devkitc_1',
85-
'/dev/boards/esp32_s2_devkitc_1',
86-
'build_esp_bsp_generic',
87-
id='esp_bsp_generic'),
88-
pytest.param('/dev/boards/esp32_s3_korvo_2',
89-
'/dev/boards/esp32_s3_korvo_2',
90-
'build_esp32_s3_korvo_2',
91-
id='esp32_s3_korvo_2'),
92-
pytest.param('/dev/boards/esp32_p4_eye',
93-
'/dev/boards/esp32_p4_eye',
94-
'build_esp32_p4_eye',
95-
id='esp32_p4_eye'),
96-
]
9710

98-
# filter by markers
99-
selected_params = [
100-
p for p in all_params if p.id in allowed_ids
101-
]
11+
def pytest_generate_tests(metafunc):
12+
test_parameters = []
10213

103-
# if not markers, use all
104-
if not selected_params:
105-
selected_params = all_params
14+
for marker in metafunc.definition.iter_markers():
15+
if "runner_" in marker.name:
16+
bsp_name = marker.name.removeprefix("runner_")
17+
bsp_param = pytest.param(
18+
f'/dev/boards/{bsp_name}',
19+
f'/dev/boards/{bsp_name}',
20+
f'build_{bsp_name}',
21+
# Test are ran one at a time for each board
22+
marks=pytest.mark.xdist_group(name=f'{bsp_name}')
23+
)
24+
test_parameters.append(bsp_param)
10625

10726
if 'port' in metafunc.fixturenames and 'flash_port' in metafunc.fixturenames:
10827
metafunc.parametrize(
10928
'port, flash_port, build_dir',
110-
selected_params
29+
test_parameters
11130
)
11231

11332

‎examples/display/pytest_display.py‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,25 @@
1-
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
1+
# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD
22
# SPDX-License-Identifier: CC0-1.0
33

44
import pytest
55
from pytest_embedded import Dut
66

77

8-
@pytest.mark.esp_box_3
9-
@pytest.mark.esp32_p4_function_ev_board
10-
@pytest.mark.esp32_c3_lcdkit
11-
@pytest.mark.esp32_s3_eye
12-
@pytest.mark.esp32_s3_lcd_ev_board
13-
@pytest.mark.esp32_s3_lcd_ev_board_2
14-
@pytest.mark.esp32_s3_usb_otg
15-
@pytest.mark.esp_wrover_kit
16-
@pytest.mark.esp32_s3_korvo_2
17-
@pytest.mark.m5dial
18-
@pytest.mark.m5stack_core
19-
@pytest.mark.m5stack_core_2
20-
@pytest.mark.m5stack_core_s3
21-
@pytest.mark.m5stack_core_s3_se
22-
@pytest.mark.m5_atom_s3
23-
@pytest.mark.esp32_p4_eye
8+
@pytest.mark.runner_esp_box_3
9+
@pytest.mark.runner_esp32_p4_function_ev_board
10+
@pytest.mark.runner_esp32_c3_lcdkit
11+
@pytest.mark.runner_esp32_s3_eye
12+
@pytest.mark.runner_esp32_s3_lcd_ev_board
13+
@pytest.mark.runner_esp32_s3_lcd_ev_board_2
14+
@pytest.mark.runner_esp32_s3_usb_otg
15+
@pytest.mark.runner_esp_wrover_kit
16+
@pytest.mark.runner_esp32_s3_korvo_2
17+
@pytest.mark.runner_m5dial
18+
@pytest.mark.runner_m5stack_core
19+
@pytest.mark.runner_m5stack_core_2
20+
@pytest.mark.runner_m5stack_core_s3
21+
@pytest.mark.runner_m5_atom_s3
22+
@pytest.mark.runner_esp32_p4_eye
2423
def test_example_display(dut: Dut) -> None:
2524
dut.expect_exact('example: Display LVGL animation')
2625
dut.expect_exact('main_task: Returned from app_main()')

‎examples/display_lvgl_benchmark/pytest_display_lvgl_benchmark.py‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
1+
# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD
22
# SPDX-License-Identifier: CC0-1.0
33

44
import os
@@ -54,14 +54,13 @@ def get_test_diff(test1, test2, name, positive):
5454
return f"*<span style=\"color:{color}\"><sub>({sign}{diff})</sub></span>*"
5555

5656

57-
@pytest.mark.esp_box_3
58-
@pytest.mark.esp32_p4_function_ev_board
59-
@pytest.mark.esp32_s3_eye
60-
@pytest.mark.esp32_s3_lcd_ev_board
61-
@pytest.mark.esp32_s3_lcd_ev_board_2
62-
@pytest.mark.m5dial
63-
@pytest.mark.m5stack_core_s3
64-
@pytest.mark.m5stack_core_s3_se
57+
@pytest.mark.runner_esp_box_3
58+
@pytest.mark.runner_esp32_p4_function_ev_board
59+
@pytest.mark.runner_esp32_s3_eye
60+
@pytest.mark.runner_esp32_s3_lcd_ev_board
61+
@pytest.mark.runner_esp32_s3_lcd_ev_board_2
62+
@pytest.mark.runner_m5dial
63+
@pytest.mark.runner_m5stack_core_s3
6564
def test_example_lvgl_benchmark(dut: Dut, request) -> None:
6665
date = datetime.datetime.now()
6766
board = request.node.callspec.id

‎examples/display_lvgl_demos/pytest_display_lvgl_demos.py‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,17 @@
1-
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
1+
# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD
22
# SPDX-License-Identifier: CC0-1.0
33

44
import pytest
55
from pytest_embedded import Dut
66

77

8-
@pytest.mark.esp_box_3
9-
@pytest.mark.esp32_p4_function_ev_board
10-
@pytest.mark.esp32_s3_eye
11-
@pytest.mark.esp32_s3_lcd_ev_board
12-
@pytest.mark.esp32_s3_lcd_ev_board_2
13-
@pytest.mark.m5dial
14-
@pytest.mark.m5stack_core_s3
15-
@pytest.mark.m5stack_core_s3_se
8+
@pytest.mark.runner_esp_box_3
9+
@pytest.mark.runner_esp32_p4_function_ev_board
10+
@pytest.mark.runner_esp32_s3_eye
11+
@pytest.mark.runner_esp32_s3_lcd_ev_board
12+
@pytest.mark.runner_esp32_s3_lcd_ev_board_2
13+
@pytest.mark.runner_m5dial
14+
@pytest.mark.runner_m5stack_core_s3
1615
def test_example_lvgl_demos(dut: Dut) -> None:
1716
dut.expect_exact('app_main: Display LVGL demo')
1817
dut.expect_exact('main_task: Returned from app_main()')

‎examples/display_sdcard/pytest_sdcard.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from pytest_embedded import Dut
66

77

8-
@pytest.mark.esp32_p4_function_ev_board
9-
@pytest.mark.esp32_s3_korvo_2
8+
@pytest.mark.runner_esp32_p4_function_ev_board
9+
@pytest.mark.runner_esp32_s3_korvo_2
1010
def test_example_sdcard(dut: Dut) -> None:
1111
dut.expect_exact('example: Testing of SD card passed!')

‎examples/generic_button_led/pytest_generic_button_led.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from pytest_embedded import Dut
66

77

8-
@pytest.mark.esp_bsp_devkit
9-
@pytest.mark.esp_bsp_generic
8+
@pytest.mark.runner_esp_bsp_devkit
9+
@pytest.mark.runner_esp_bsp_generic
1010
def test_example_generic_button_led(dut: Dut) -> None:
1111
dut.expect_exact('main_task: Returned from app_main()')

‎pytest.ini‎

Lines changed: 28 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,10 @@
11
[pytest]
22
# only the files with prefix `pytest_` would be recognized as pytest test scripts.
33
python_files = pytest_*.py
4-
4+
strict_markers = true
55
# set traceback to "short" to prevent the overwhelming tracebacks
6-
addopts = -s --embedded-services esp,idf --tb short
7-
markers =
8-
# env markers
9-
esp_box_3: esp-box-3 runner
10-
esp32_c3_lcdkit: esp32_c3_lcdkit runner
11-
esp32_p4_box: esp32_p4_box runner
12-
esp32_p4_function_ev_board: esp32_p4_function_ev_board runner
13-
esp32_s3_eye: esp32_s3_eye runner
14-
esp32_s3_lcd_ev_board: esp32_s3_lcd_ev_board runner
15-
esp32_s3_lcd_ev_board_2: esp32_s3_lcd_ev_board runner
16-
esp32_s3_usb_otg: esp32_s3_usb_otg runner
17-
esp_wrover_kit: esp_wrover_kit runner
18-
m5dial: m5dial runner
19-
m5stack_core: m5stack_core runner
20-
m5stack_core_2: m5stack_core_2 runner
21-
m5stack_core_s3: m5stack_core_s3 runner
22-
m5stack_core_s3_se: m5stack_core_s3_se runner
23-
esp32_azure_iot_kit: esp32_azure_iot_kit runner
24-
esp_bsp_devkit: esp32_s3_devkitc_1_1 runner
25-
esp_bsp_generic: esp32_s3_devkitc_1_1 runner
26-
esp32_s3_korvo_2: esp32_s3_korvo_2 runner
27-
m5_atom_s3: m5 Atom S3 runner
28-
6+
tb = short
7+
norecursedirs = .github managed_components
298
# log related
309
log_cli = True
3110
log_cli_level = INFO
@@ -36,3 +15,28 @@ junit_family = xunit1
3615
## log all to `system-out` when case fail
3716
junit_logging = stdout
3817
junit_log_passing_tests = False
18+
19+
addopts = -s --embedded-services esp,idf -p no:idf-ci --suppress-no-test-exit-code
20+
21+
markers =
22+
# env markers
23+
runner_esp_box_3: esp-box-3 runner
24+
runner_esp32_c3_lcdkit: esp32_c3_lcdkit runner
25+
runner_esp32_p4_box: esp32_p4_box runner
26+
runner_esp32_p4_function_ev_board: esp32_p4_function_ev_board runner
27+
runner_esp32_s3_eye: esp32_s3_eye runner
28+
runner_esp32_s3_lcd_ev_board: esp32_s3_lcd_ev_board runner
29+
runner_esp32_s3_lcd_ev_board_2: esp32_s3_lcd_ev_board runner
30+
runner_esp32_s3_usb_otg: esp32_s3_usb_otg runner
31+
runner_esp_wrover_kit: esp_wrover_kit runner
32+
runner_m5dial: m5dial runner
33+
runner_m5stack_core: m5stack_core runner
34+
runner_m5stack_core_2: m5stack_core_2 runner
35+
runner_m5stack_core_s3: m5stack_core_s3 runner
36+
runner_m5stack_core_s3_se: m5stack_core_s3_se runner
37+
runner_esp32_azure_iot_kit: esp32_azure_iot_kit runner
38+
runner_esp_bsp_devkit: esp32_s3_devkitc_1_1 runner
39+
runner_esp_bsp_generic: esp32_s3_devkitc_1_1 runner
40+
runner_esp32_s3_korvo_2: esp32_s3_korvo_2 runner
41+
runner_m5_atom_s3: m5 Atom S3 runner
42+
runner_esp32_p4_eye: ESP32-P4 EYE runner

0 commit comments

Comments
 (0)
⚡