ESP-BSP benchmark action #1781
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build ESP-BSP apps | |
| # This job builds all examples and test_applications in this repo | |
| # Applications are selected for build based on changes files and dependencies defined in .build-test-rules.yml | |
| on: | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| paths-ignore: | |
| - '*.md' | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - '*.md' | |
| schedule: | |
| - cron: '0 0 * * *' # every day at midnight | |
| workflow_dispatch: | |
| # Cancel previous CI, if running and changed label or pushed PR (Prevent to wait for runners) | |
| concurrency: | |
| group: pr-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| # Add comments to PR | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| checks: write | |
| jobs: | |
| build: | |
| outputs: | |
| ignored_failure: ${{ steps.ignored-failure.outputs.failed }} | |
| strategy: | |
| fail-fast: ${{ github.event_name != 'schedule' }} | |
| matrix: | |
| idf_ver: | |
| - "latest" | |
| - "release-v6.0" | |
| - "release-v5.5" | |
| - "release-v5.4" | |
| - "release-v5.3" | |
| - "release-v5.2" | |
| parallel_index: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] | |
| include: | |
| # Default values for all versions | |
| - parallel_count: 10 | |
| allow_fail: false | |
| - idf_ver: "latest" | |
| allow_fail: true # Do not fail CI, when fail "latest" build | |
| - idf_ver: "release-v6.0" | |
| allow_fail: true # Do not fail CI, when fail "latest" build | |
| runs-on: ubuntu-latest | |
| container: espressif/idf:${{ matrix.idf_ver }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Get changed files | |
| if: github.event_name == 'pull_request' # This action is working only in pull-request (it fails in push to master) | |
| id: changed-files | |
| uses: tj-actions/changed-files@v47 | |
| with: | |
| separator: ';' # idf-build-apps expects files seprated with semicolon | |
| - name: Build ESP-BSP applications | |
| id: buildapps | |
| continue-on-error: ${{ github.event_name == 'pull_request' && matrix.allow_fail == true }} | |
| if: always() | |
| shell: bash | |
| env: | |
| IDF_EXTRA_ACTIONS_PATH: "${{ github.workspace }}/examples" | |
| # When event not a pull-request, the action before must be skipped and the steps.changed-files.outputs.all_modified_files variable is not exists and must be removed | |
| ALL_CHANGED_FILES: ${{ (github.event_name != 'pull_request') && format(' ') || format('--modified-files=') }}${{ (github.event_name != 'pull_request') && format(' ') || steps.changed-files.outputs.all_modified_files }} | |
| run: | | |
| export IDF_PYTHON_CHECK_CONSTRAINTS=yes | |
| ${IDF_PATH}/install.sh --enable-ci | |
| source ${IDF_PATH}/export.sh | |
| echo "Files changed: "${{env.ALL_CHANGED_FILES}} | |
| idf-build-apps find --path . --target all ${{env.ALL_CHANGED_FILES}} | |
| idf-build-apps build --path . --target all --parallel-count ${{ matrix.parallel_count }} --parallel-index ${{ matrix.parallel_index }} ${{env.ALL_CHANGED_FILES}} --collect-app-info build_info_${{ matrix.idf_ver }}_${{ matrix.parallel_index }}.json | |
| - name: Save and write the outcome of an ignored failing job | |
| id: ignored-failure | |
| if: ${{ github.event_name == 'pull_request' && steps.buildapps.outcome == 'failure' && matrix.allow_fail == true }} | |
| run: | | |
| echo "failed=true" >> "$GITHUB_OUTPUT" | |
| echo "::warning::Build failed for idf_ver=${{ matrix.idf_ver }} (ignored because it's '${{ matrix.idf_ver }}')" | |
| echo "### ⚠️ Build failed for idf_ver=${{ matrix.idf_ver }}" >> $GITHUB_STEP_SUMMARY | |
| echo "This failure was ignored (continue-on-error enabled)." >> $GITHUB_STEP_SUMMARY | |
| - uses: actions/upload-artifact@v7 | |
| if: github.repository_owner == 'espressif' # && needs.prepare.outputs.build_only == '0' | |
| with: | |
| name: app_binaries_${{ matrix.idf_ver }}_${{ matrix.parallel_index }} | |
| path: | | |
| examples/*/build_*/bootloader/bootloader.bin | |
| examples/*/build_*/partition_table/partition-table.bin | |
| examples/*/build_*/*.bin | |
| examples/*/build_*/flasher_args.json | |
| examples/*/build_*/config/sdkconfig.json | |
| build_info*.json | |
| comment-failure: | |
| name: Comment on the PR about the ignored failure | |
| needs: build | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:3.11-bookworm | |
| env: | |
| IGNORED_FAILURE: ${{needs.build.outputs.ignored_failure}} | |
| steps: | |
| - name: Write a comment | |
| if: | | |
| env.IGNORED_FAILURE == 'true' && | |
| contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) | |
| uses: thollander/actions-comment-pull-request@v3 | |
| with: | |
| comment-tag: build_warning_latest_failed | |
| mode: recreate | |
| message: | | |
| ### ⚠️ Build failure was ignored (continue-on-error enabled). | |
| See details: [View build logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#step:build) | |
| run-target: | |
| name: Run apps | |
| if: github.repository_owner == 'espressif' && !contains(github.event.pull_request.labels.*.name, 'Build only') | |
| needs: build | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 5 | |
| matrix: | |
| idf_ver: | |
| - "release-v5.5" | |
| env: | |
| TEST_RESULT_NAME: test_results_${{ matrix.runner.example }}_${{ matrix.idf_ver }} | |
| TEST_RESULT_FILE: test_results_${{ matrix.runner.example }}_${{ matrix.idf_ver }}.xml | |
| runs-on: [self-hosted, Linux, bspwall] | |
| container: | |
| image: espressif/idf:${{ matrix.idf_ver }} | |
| options: --privileged -v /dev/boards:/dev/boards/ # Privileged mode has access to serial ports | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: app_binaries_${{ matrix.idf_ver }}_* | |
| merge-multiple: true | |
| - name: Download latest results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: benchmark_* | |
| path: benchmark/ | |
| - name: Run apps | |
| run: | | |
| export IDF_PYTHON_CHECK_CONSTRAINTS=yes | |
| ${IDF_PATH}/install.sh --enable-ci --enable-pytest | |
| . ${IDF_PATH}/export.sh | |
| pip install --prefer-binary pytest-custom_exit_code pytest-xdist | |
| export PYTEST_EMBEDDED_CACHE_DIR=/tmp/pytest-embedded-cache-dummy | |
| mkdir -p /tmp/pytest-embedded-cache-dummy | |
| pytest --junit-xml=${{ env.TEST_RESULT_FILE }} -n auto --dist loadgroup | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: ${{ env.TEST_RESULT_NAME }} | |
| path: | | |
| ${{ env.TEST_RESULT_FILE }} | |
| *.log | |
| benchmark_*.md | |
| benchmark_*.json | |
| benchmark.json | |
| - name: Update benchmark release | |
| uses: pyTooling/Actions/releaser@r0 | |
| if: github.ref_name == 'master' && env.benchmark_files_exist == 'true' | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| files: | | |
| benchmark_*.json | |
| benchmark_*.md | |
| tag: benchmark-latest | |
| publish-results: | |
| name: Publish Test results | |
| needs: | |
| - run-target | |
| if: github.repository_owner == 'espressif' && always() && github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'Build only') | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Download Test results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: test_results_* | |
| path: test_results | |
| - name: Publish Test Results | |
| uses: EnricoMi/publish-unit-test-result-action@v2 | |
| with: | |
| files: test_results/**/*.xml | |
| - name: Find benchmark result files | |
| if: (contains(github.event.pull_request.labels.*.name, 'Run benchmark') || contains(inputs.WFType, 'Build + Tests + Benchmark') || github.ref_name == 'master') | |
| id: find_files | |
| run: | | |
| OUTPUT_FILE="combined_benchmarks.md" | |
| echo "" > $OUTPUT_FILE | |
| python <<EOF | |
| import glob | |
| files = sorted(glob.glob("test_results/**/benchmark_*.md")) | |
| print(files) | |
| output_file = "combined_benchmarks.md" | |
| with open(output_file, "w", encoding="utf-8") as outfile: | |
| for file in files: | |
| with open(file, "r", encoding="utf-8") as infile: | |
| outfile.write(infile.read() + "\n\n") | |
| print(f"Merged {len(files)} files into {output_file}") | |
| EOF | |
| echo "output_file=$OUTPUT_FILE" >> "$GITHUB_ENV" | |
| - name: Comment PR | |
| if: (contains(github.event.pull_request.labels.*.name, 'Run benchmark') || contains(inputs.WFType, 'Build + Tests + Benchmark') || github.ref_name == 'master') | |
| uses: thollander/actions-comment-pull-request@v3 | |
| with: | |
| comment-tag: benchmark_results | |
| file-path: ${{ env.output_file }} |