Merge pull request #564 from espressif/ci/parallel_runner

espzav · web-flow · commit d7ada305bfd7 · 2025-06-25T11:30:39.000+02:00
ci(runner): Process runner in parallel
diff --git a/.github/workflows/build-run-applications.yml b/.github/workflows/build-run-applications.yml
@@ -71,6 +71,12 @@ jobs:
           - idf_ver: "release-v5.4"
             parallel_count: 2
             parallel_index: 2
+          - idf_ver: "release-v5.5"
+            parallel_count: 2
+            parallel_index: 1
+          - idf_ver: "release-v5.5"
+            parallel_count: 2
+            parallel_index: 2
     runs-on: ubuntu-latest
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -121,67 +127,15 @@ jobs:
         idf_ver:
           - "latest"
         runner:
-          - runs-on: "esp-box-3"
-            marker: "esp_box_3"
-            target: "esp32s3"
-          - runs-on: "esp32_c3_lcdkit"
-            marker: "esp32_c3_lcdkit"
-            target: "esp32c3"
-          - runs-on: "esp32_p4_box"
-            marker: "esp32_p4_box"
-            target: "esp32p4"
-          - runs-on: "esp32_p4_function_ev_board"
-            marker: "esp32_p4_function_ev_board"
-            target: "esp32p4"
-          - runs-on: "esp32_s3_eye"
-            marker: "esp32_s3_eye"
-            target: "esp32s3"
-          - runs-on: "esp32_s3_lcd_ev_board"
-            marker: "esp32_s3_lcd_ev_board"
-            target: "esp32s3"
-          - runs-on: "esp32_s3_lcd_ev_board"
-            marker: "esp32_s3_lcd_ev_board_2"
-            target: "esp32s3"
-          - runs-on: "esp32_s3_usb_otg"
-            marker: "esp32_s3_usb_otg"
-            target: "esp32s3"
-          - runs-on: "esp_wrover_kit"
-            marker: "esp_wrover_kit"
-            target: "esp32"
-          - runs-on: "m5dial"
-            marker: "m5dial"
-            target: "esp32s3"
-          - runs-on: "m5stack_core"
-            marker: "m5stack_core"
-            target: "esp32"
-          - runs-on: "m5stack_core_2"
-            marker: "m5stack_core_2"
-            target: "esp32"
-          - runs-on: "m5stack_core_s3"
-            marker: "m5stack_core_s3"
-            target: "esp32s3"
-          - runs-on: "m5stack_core_s3"
-            marker: "m5stack_core_s3_se"
-            target: "esp32s3"
-          - runs-on: "esp32_azure_iot_kit"
-            marker: "esp32_azure_iot_kit"
-            target: "esp32"
-          - runs-on: "esp_bsp_devkit"
-            marker: "esp_bsp_devkit"
-            target: "esp32s3"
-          - runs-on: "esp_bsp_generic"
-            marker: "esp_bsp_generic"
-            target: "esp32s3"
-          - runs-on: "esp32_s3_korvo_2"
-            marker: "esp32_s3_korvo_2"
-            target: "esp32s3"
-          - runs-on: "m5_atom_s3"
-            marker: "m5_atom_s3"
-            target: "esp32s3"
+          - example: "test_example_display"
+          - example: "test_example_lvgl_demos"
+          - example: "test_example_display_sensors"
+          - example: "test_example_generic_button_led"
+          - example: "test_example_lvgl_benchmark"
     env:
-      TEST_RESULT_NAME: test_results_${{ matrix.runner.target }}_${{ matrix.runner.marker }}_${{ matrix.idf_ver }}
-      BENCHMARK_RESULT_NAME: benchmark_${{ matrix.runner.target }}_${{ matrix.runner.marker }}_${{ matrix.idf_ver }}
-      TEST_RESULT_FILE: test_results_${{ matrix.runner.target }}_${{ matrix.runner.marker }}_${{ matrix.idf_ver }}.xml
+      TEST_RESULT_NAME: test_results_${{ matrix.runner.example }}_${{ matrix.idf_ver }}
+      BENCHMARK_RESULT_NAME: benchmark_${{ matrix.runner.example }}_${{ matrix.idf_ver }}
+      TEST_RESULT_FILE: test_results_${{ matrix.runner.example }}_${{ matrix.idf_ver }}.xml
       PYTEST_BENCHMARK_IGNORE: ${{ (contains(github.event.pull_request.labels.*.name, 'Run benchmark') || contains(inputs.WFType, 'Build + Tests + Benchmark') || github.ref_name == 'master') && format(' ') || format('--ignore=examples/display_lvgl_benchmark') }}
     runs-on: [self-hosted, Linux, bspwall]
     container:
@@ -197,15 +151,17 @@ jobs:
         env:
           PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi/"
         run: |
-          pip install --prefer-binary cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code
+          pip install --prefer-binary cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code pytest-xdist
       - name: Download latest results
         uses: actions/download-artifact@v4
         with:
           pattern: benchmark_*
           path: benchmark/
       - name: Run apps
         run: |
-          pytest --suppress-no-test-exit-code --ignore-glob '*/managed_components/*' --ignore=.github --junit-xml=${{ env.TEST_RESULT_FILE }} --target=${{ matrix.runner.target }} -m ${{ matrix.runner.marker }} --build-dir=build_${{ matrix.runner.runs-on }} ${{ env.PYTEST_BENCHMARK_IGNORE }}
+          export PYTEST_EMBEDDED_CACHE_DIR=/tmp/pytest-embedded-cache-dummy
+          mkdir -p /tmp/pytest-embedded-cache-dummy
+          pytest --suppress-no-test-exit-code --ignore-glob '*/managed_components/*' --ignore=.github --junit-xml=${{ env.TEST_RESULT_FILE }} -k ${{ matrix.runner.example }} -n auto ${{ env.PYTEST_BENCHMARK_IGNORE }}
       - name: Upload test results
         uses: actions/upload-artifact@v4
         if: always()
diff --git a/conftest.py b/conftest.py
@@ -1,40 +1,123 @@
-# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
+import os
+import uuid
 import pytest
 
 
 def pytest_generate_tests(metafunc):
-    # If test expects param 'port' or 'flash_port', add parametrize
+    allowed_ids = set()
+
+    # get markers from test case (e.g. 'esp_box_3')
+    for marker in metafunc.definition.iter_markers():
+        allowed_ids.add(marker.name)
+
+    all_params = [
+        pytest.param('/dev/boards/esp-box-3',
+                     '/dev/boards/esp-box-3',
+                     'build_esp-box-3',
+                     id='esp_box_3'),
+        pytest.param('/dev/boards/esp32_c3_lcdkit',
+                     '/dev/boards/esp32_c3_lcdkit',
+                     'build_esp32_c3_lcdkit',
+                     id='esp32_c3_lcdkit'),
+        pytest.param('/dev/boards/esp32_p4_box',
+                     '/dev/boards/esp32_p4_box',
+                     'build_esp32_p4_box',
+                     id='esp32_p4_box'),
+        pytest.param('/dev/boards/esp32_p4_function_ev_board',
+                     '/dev/boards/esp32_p4_function_ev_board',
+                     'build_esp32_p4_function_ev_board',
+                     id='esp32_p4_function_ev_board'),
+        pytest.param('/dev/boards/esp32_s3_eye',
+                     '/dev/boards/esp32_s3_eye',
+                     'build_esp32_s3_eye',
+                     id='esp32_s3_eye'),
+        pytest.param('/dev/boards/esp32_s3_lcd_ev_board',
+                     '/dev/boards/esp32_s3_lcd_ev_board',
+                     'build_esp32_s3_lcd_ev_board',
+                     id='esp32_s3_lcd_ev_board'),
+        pytest.param('/dev/boards/esp32_s3_lcd_ev_board-2',
+                     '/dev/boards/esp32_s3_lcd_ev_board-2',
+                     'build_esp32_s3_lcd_ev_board',
+                     id='esp32_s3_lcd_ev_board_2'),
+        pytest.param('/dev/boards/esp32_s3_usb_otg',
+                     '/dev/boards/esp32_s3_usb_otg',
+                     'build_esp32_s3_usb_otg',
+                     id='esp32_s3_usb_otg'),
+        pytest.param('/dev/boards/esp_wrover_kit',
+                     '/dev/boards/esp_wrover_kit',
+                     'build_esp_wrover_kit',
+                     id='esp_wrover_kit'),
+        pytest.param('/dev/boards/esp32_azure_iot_kit',
+                     '/dev/boards/esp32_azure_iot_kit',
+                     'build_esp32_azure_iot_kit',
+                     id='esp32_azure_iot_kit'),
+        pytest.param('/dev/boards/m5dial',
+                     '/dev/boards/m5dial',
+                     'build_m5dial',
+                     id='m5dial'),
+        pytest.param('/dev/boards/m5stack_core',
+                     '/dev/boards/m5stack_core',
+                     'build_m5stack_core',
+                     id='m5stack_core'),
+        pytest.param('/dev/boards/m5stack_core_2',
+                     '/dev/boards/m5stack_core_2',
+                     'build_m5stack_core',
+                     id='m5stack_core_2'),
+        pytest.param('/dev/boards/m5stack_core_s3',
+                     '/dev/boards/m5stack_core_s3',
+                     'build_m5stack_core_s3',
+                     id='m5stack_core_s3'),
+        pytest.param('/dev/boards/m5stack_core_s3_se',
+                     '/dev/boards/m5stack_core_s3_se',
+                     'build_m5stack_core_s3',
+                     id='m5stack_core_s3_se'),
+        pytest.param('/dev/boards/m5_atom_s3',
+                     '/dev/boards/m5_atom_s3',
+                     'build_m5_atom_s3',
+                     id='m5_atom_s3'),
+        pytest.param('/dev/boards/esp32_s3_devkitc_1_1',
+                     '/dev/boards/esp32_s3_devkitc_1_1',
+                     'build_esp_bsp_devkit',
+                     id='esp_bsp_devkit'),
+        pytest.param('/dev/boards/esp32_s2_devkitc_1',
+                     '/dev/boards/esp32_s2_devkitc_1',
+                     'build_esp_bsp_generic',
+                     id='esp_bsp_generic'),
+        pytest.param('/dev/boards/esp32_s3_korvo_2',
+                     '/dev/boards/esp32_s3_korvo_2',
+                     'build_esp32_s3_korvo_2',
+                     id='esp32_s3_korvo_2'),
+    ]
+
+    # filter by markers
+    selected_params = [
+        p for p in all_params if p.id in allowed_ids
+    ]
+
+    # if not markers, use all
+    if not selected_params:
+        selected_params = all_params
+
     if 'port' in metafunc.fixturenames and 'flash_port' in metafunc.fixturenames:
         metafunc.parametrize(
-            'port, flash_port',
-            [
-                pytest.param('/dev/boards/esp-box-3', '/dev/boards/esp-box-3', id='esp_box_3'),
-                pytest.param('/dev/boards/esp32_c3_lcdkit', '/dev/boards/esp32_c3_lcdkit', id='esp32_c3_lcdkit'),
-                pytest.param('/dev/boards/esp32_p4_box', '/dev/boards/esp32_p4_box', id='esp32_p4_box'),
-                pytest.param('/dev/boards/esp32_p4_function_ev_board', '/dev/boards/esp32_p4_function_ev_board', id='esp32_p4_function_ev_board'),
-                pytest.param('/dev/boards/esp32_s3_eye', '/dev/boards/esp32_s3_eye', id='esp32_s3_eye'),
-                pytest.param('/dev/boards/esp32_s3_lcd_ev_board', '/dev/boards/esp32_s3_lcd_ev_board', id='esp32_s3_lcd_ev_board'),
-                pytest.param('/dev/boards/esp32_s3_lcd_ev_board-2', '/dev/boards/esp32_s3_lcd_ev_board-2', id='esp32_s3_lcd_ev_board_2'),
-                pytest.param('/dev/boards/esp32_s3_usb_otg', '/dev/boards/esp32_s3_usb_otg', id='esp32_s3_usb_otg'),
-                pytest.param('/dev/boards/esp_wrover_kit', '/dev/boards/esp_wrover_kit', id='esp_wrover_kit'),
-                pytest.param('/dev/boards/esp32_azure_iot_kit', '/dev/boards/esp32_azure_iot_kit', id='esp32_azure_iot_kit'),
-                pytest.param('/dev/boards/m5dial', '/dev/boards/m5dial', id='m5dial'),
-                pytest.param('/dev/boards/m5stack_core', '/dev/boards/m5stack_core', id='m5stack_core'),
-                pytest.param('/dev/boards/m5stack_core_2', '/dev/boards/m5stack_core_2', id='m5stack_core_2'),
-                pytest.param('/dev/boards/m5stack_core_s3', '/dev/boards/m5stack_core_s3', id='m5stack_core_s3'),
-                pytest.param('/dev/boards/m5stack_core_s3_se', '/dev/boards/m5stack_core_s3_se', id='m5stack_core_s3_se'),
-                pytest.param('/dev/boards/m5_atom_s3', '/dev/boards/m5_atom_s3', id='m5_atom_s3'),
-                pytest.param('/dev/boards/esp32_s3_devkitc_1_1', '/dev/boards/esp32_s3_devkitc_1_1', id='esp_bsp_devkit'),
-                pytest.param('/dev/boards/esp32_s3_devkitc_1_1', '/dev/boards/esp32_s3_devkitc_1_1', id='esp_bsp_generic'),
-                pytest.param('/dev/boards/esp32_s3_korvo_2', '/dev/boards/esp32_s3_korvo_2', id='esp32_s3_korvo_2'),
-            ]
+            'port, flash_port, build_dir',
+            selected_params
         )
 
 
-# Check marker with params and test ID
-def pytest_collection_modifyitems(config, items):
-    for item in items:
-        marker_option = "[" + config.getoption("-m") + "]"
-        if marker_option not in item.nodeid:
-            item.add_marker(pytest.mark.skip(reason="Not for selected params"))
+@pytest.fixture
+def build_dir(build_dir: str) -> str:
+    return f'{build_dir}'
+
+
+# This fixing using cache when used "-n auto" (parallel)
+def pytest_configure(config):
+    # If run pytest-xdist (parallel), set unique cache dir
+    worker_id = os.getenv("PYTEST_XDIST_WORKER", None)
+    if worker_id:
+        cache_dir = f"/tmp/pytest-embedded-cache-{uuid.uuid4()}"
+        os.environ["PYTEST_EMBEDDED_CACHE_DIR"] = cache_dir
+        os.makedirs(cache_dir, exist_ok=True)
+        print(f"Using embedded cache dir: {cache_dir}")
diff --git a/examples/display/pytest_display.py b/examples/display/pytest_display.py
@@ -20,6 +20,6 @@
 @pytest.mark.m5stack_core_s3
 @pytest.mark.m5stack_core_s3_se
 @pytest.mark.m5_atom_s3
-def test_display_example(dut: Dut) -> None:
+def test_example_display(dut: Dut) -> None:
     dut.expect_exact('example: Display LVGL animation')
     dut.expect_exact('main_task: Returned from app_main()')
diff --git a/examples/display_lvgl_benchmark/main/main.c b/examples/display_lvgl_benchmark/main/main.c
@@ -24,7 +24,6 @@ static char *TAG = "app_main";
 void app_main(void)
 {
     /* Initialize display and LVGL */
-
 #if defined(BSP_BOARD_ESP32_S3_LCD_EV_BOARD)
     /* Only for esp32_s3_lcd_ev_board */
     bsp_display_cfg_t cfg = {
diff --git a/examples/display_lvgl_benchmark/pytest_display_lvgl_benchmark.py b/examples/display_lvgl_benchmark/pytest_display_lvgl_benchmark.py
@@ -62,7 +62,7 @@ def get_test_diff(test1, test2, name, positive):
 @pytest.mark.m5dial
 @pytest.mark.m5stack_core_s3
 @pytest.mark.m5stack_core_s3_se
-def test_example(dut: Dut, request) -> None:
+def test_example_lvgl_benchmark(dut: Dut, request) -> None:
     date = datetime.datetime.now()
     board = request.node.callspec.id
 
diff --git a/examples/display_lvgl_demos/pytest_display_lvgl_demos.py b/examples/display_lvgl_demos/pytest_display_lvgl_demos.py
@@ -13,6 +13,6 @@
 @pytest.mark.m5dial
 @pytest.mark.m5stack_core_s3
 @pytest.mark.m5stack_core_s3_se
-def test_display_example(dut: Dut) -> None:
+def test_example_lvgl_demos(dut: Dut) -> None:
     dut.expect_exact('app_main: Display LVGL demo')
     dut.expect_exact('main_task: Returned from app_main()')
diff --git a/examples/display_sensors/pytest_display_display_sensors.py b/examples/display_sensors/pytest_display_display_sensors.py
@@ -6,6 +6,6 @@
 
 
 @pytest.mark.esp32_azure_iot_kit
-def test_display_example(dut: Dut) -> None:
+def test_example_display_sensors(dut: Dut) -> None:
     dut.expect_exact('main_task: Returned from app_main()')
     dut.expect(r'example: temperature: (\d+[.]\d+), humidity: (\d+[.]\d+), luminance: (\d+[.]\d+)')
diff --git a/examples/generic_button_led/pytest_generic_button_led.py b/examples/generic_button_led/pytest_generic_button_led.py
@@ -7,6 +7,6 @@
 
 @pytest.mark.esp_bsp_devkit
 @pytest.mark.esp_bsp_generic
-def test_generic_button_led(dut: Dut) -> None:
+def test_example_generic_button_led(dut: Dut) -> None:
     dut.expect_exact('led_indicator: Indicator create successfully.')
     dut.expect_exact('main_task: Returned from app_main()')
diff --git a/examples/generic_button_led/sdkconfig.bsp.esp_bsp_generic b/examples/generic_button_led/sdkconfig.bsp.esp_bsp_generic
@@ -1,11 +1,11 @@
 # This file was generated using idf.py save-defconfig. It can be edited manually.
 # Espressif IoT Development Framework (ESP-IDF) Project Minimal Configuration
 #
-CONFIG_IDF_TARGET="esp32s3"
+CONFIG_IDF_TARGET="esp32s2"
 CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
 CONFIG_PARTITION_TABLE_CUSTOM=y
 
-# ESP32-S3-DevKitC-1.1 Settings
+# ESP32-S2-DevKitC-1 Settings
 # Buttons
 CONFIG_BSP_BUTTONS_NUM=1
 CONFIG_BSP_BUTTON_1_TYPE_GPIO=y
@@ -14,5 +14,5 @@ CONFIG_BSP_BUTTON_1_LEVEL=0
 # LEDs
 CONFIG_BSP_LEDS_NUM=1
 CONFIG_BSP_LED_TYPE_RGB=y
-CONFIG_BSP_LED_RGB_GPIO=38
+CONFIG_BSP_LED_RGB_GPIO=18
 CONFIG_BSP_LED_RGB_BACKEND_RMT=y

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,6 @@ static char *TAG = "app_main";`
`24`	`24`	`void app_main(void)`
`25`	`25`	`{`
`26`	`26`	`/* Initialize display and LVGL */`
`27`		`-`
`28`	`27`	`#if defined(BSP_BOARD_ESP32_S3_LCD_EV_BOARD)`
`29`	`28`	`/* Only for esp32_s3_lcd_ev_board */`
`30`	`29`	`bsp_display_cfg_t cfg = {`