Benchmark Baseline vs Current #130
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Baseline vs Current | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| baselineVersion: | |
| description: 'NuGet version of Humanizer to use for the baseline run' | |
| required: true | |
| type: string | |
| default: '3.0.10' | |
| push: | |
| paths: | |
| - '.github/workflows/benchmarks-baseline-vs-current.yml' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/benchmarks-baseline-vs-current.yml' | |
| schedule: | |
| - cron: '0 4 * * 1' # Every Monday at 04:00 UTC | |
| jobs: | |
| benchmark: | |
| name: Run Benchmarks (${{ matrix.kind }}, ${{ matrix.targetFramework }}) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| env: | |
| BASELINE_VERSION: ${{ inputs.baselineVersion || '3.0.10' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| kind: [baseline, current] | |
| targetFramework: [net8.0, net10.0, net11.0] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup .NET 8 | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| dotnet-version: '8.0.x' | |
| - name: Setup .NET 10 | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| dotnet-version: '10.0.x' | |
| - name: Setup .NET 11 preview | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| dotnet-version: '11.0.x' | |
| dotnet-quality: preview | |
| - name: Display run information | |
| run: | | |
| echo "## Benchmark Run: ${{ matrix.kind }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Commit SHA:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Baseline Version:** ${{ env.BASELINE_VERSION }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Run Type:** ${{ matrix.kind }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Target Framework:** ${{ matrix.targetFramework }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| - name: Build Benchmarks (baseline) | |
| if: matrix.kind == 'baseline' | |
| run: | | |
| out="artifacts/baseline/${{ matrix.targetFramework }}" | |
| mkdir -p "$out" | |
| dotnet run --project src/Benchmarks/Benchmarks.csproj -c Release -f ${{ matrix.targetFramework }} -p:UseBaselinePackage=true -p:BaselinePackageVersion=${{ env.BASELINE_VERSION }} -- --filter '*' --artifacts "$out" | |
| - name: Build Benchmarks (current) | |
| if: matrix.kind == 'current' | |
| run: | | |
| out="artifacts/current/${{ matrix.targetFramework }}" | |
| mkdir -p "$out" | |
| dotnet run --project src/Benchmarks/Benchmarks.csproj -c Release -f ${{ matrix.targetFramework }} -- --filter '*' --artifacts "$out" | |
| - name: Append Results to Summary | |
| run: | | |
| out="artifacts/${{ matrix.kind }}/${{ matrix.targetFramework }}" | |
| # Find all markdown files and append them to the summary | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Benchmark Results (${{ matrix.kind }}, ${{ matrix.targetFramework }})" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| while IFS= read -r MD; do | |
| cat "$MD" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| done < <(find "$out" -type f -name '*.md') | |
| - name: Upload JSON Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: humanizer-bdn-${{ matrix.kind }}-${{ matrix.targetFramework }}-json | |
| path: artifacts/${{ matrix.kind }}/${{ matrix.targetFramework }}/**/*.json | |
| retention-days: 7 | |
| - name: Upload All Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: humanizer-bdn-${{ matrix.kind }}-${{ matrix.targetFramework }}-all | |
| path: artifacts/${{ matrix.kind }}/${{ matrix.targetFramework }}/ | |
| retention-days: 7 | |
| compare: | |
| name: Compare Baseline vs Current (${{ matrix.targetFramework }}) | |
| runs-on: ubuntu-latest | |
| needs: benchmark | |
| env: | |
| BASELINE_VERSION: ${{ inputs.baselineVersion || '3.0.10' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| targetFramework: [net8.0, net10.0, net11.0] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download Baseline JSON | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: humanizer-bdn-baseline-${{ matrix.targetFramework }}-json | |
| path: baseline | |
| - name: Download Current JSON | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: humanizer-bdn-current-${{ matrix.targetFramework }}-json | |
| path: current | |
| - name: Setup .NET 8 | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| dotnet-version: '8.0.x' | |
| - name: Setup .NET 10 | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| dotnet-version: '10.0.x' | |
| - name: Setup .NET 11 preview | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| dotnet-version: '11.0.x' | |
| dotnet-quality: preview | |
| - name: Clone and Build ResultsComparer | |
| run: | | |
| git clone --depth 1 https://github.com/dotnet/performance "$RUNNER_TEMP/performance" | |
| { | |
| echo '<configuration>' | |
| echo ' <packageSources>' | |
| echo ' <clear />' | |
| echo ' <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />' | |
| echo ' <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />' | |
| echo ' </packageSources>' | |
| echo '</configuration>' | |
| } > "$RUNNER_TEMP/performance-nuget.config" | |
| dotnet restore "$RUNNER_TEMP/performance/src/tools/ResultsComparer/ResultsComparer.csproj" \ | |
| -p:PERFLAB_TARGET_FRAMEWORKS=net10.0 \ | |
| --configfile "$RUNNER_TEMP/performance-nuget.config" | |
| - name: Locate Benchmark Results | |
| id: results | |
| run: | | |
| set -euo pipefail | |
| baseline_results=$(find baseline -type d -name results -print -quit) | |
| current_results=$(find current -type d -name results -print -quit) | |
| if [ -z "$baseline_results" ]; then | |
| echo "Baseline results directory not found under 'baseline'." >&2 | |
| exit 1 | |
| fi | |
| if [ -z "$current_results" ]; then | |
| echo "Current results directory not found under 'current'." >&2 | |
| exit 1 | |
| fi | |
| echo "Using baseline results directory: $baseline_results" | |
| echo "Using current results directory: $current_results" | |
| echo "baseline_results=$baseline_results" >> "$GITHUB_OUTPUT" | |
| echo "current_results=$current_results" >> "$GITHUB_OUTPUT" | |
| - name: Run ResultsComparer for each TFM | |
| run: | | |
| mkdir -p comparisons | |
| tfm="${{ matrix.targetFramework }}" | |
| echo "Comparing results for $tfm..." | |
| compare_exit=0 | |
| dotnet run --no-restore --project "$RUNNER_TEMP/performance/src/tools/ResultsComparer/ResultsComparer.csproj" -c Release -p:PERFLAB_TARGET_FRAMEWORKS=net10.0 -f net10.0 --base "${{ steps.results.outputs.baseline_results }}" --diff "${{ steps.results.outputs.current_results }}" --threshold "5%" > "comparisons/diff-$tfm.md" || compare_exit=$? | |
| if [ ! -s "comparisons/diff-$tfm.md" ] || grep -q "The build failed" "comparisons/diff-$tfm.md"; then | |
| cat "comparisons/diff-$tfm.md" >&2 || true | |
| if [ "$compare_exit" -ne 0 ]; then | |
| exit "$compare_exit" | |
| fi | |
| exit 1 | |
| fi | |
| if [ "$compare_exit" -ne 0 ]; then | |
| echo "ResultsComparer exited with code $compare_exit; uploaded comparison report for inspection." | |
| fi | |
| - name: Append Comparison to Summary | |
| run: | | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Performance Comparison: Baseline vs Current" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Baseline Version:** ${{ env.BASELINE_VERSION }}" >> $GITHUB_STEP_SUMMARY | |
| echo "**Target Framework:** ${{ matrix.targetFramework }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Append comparison for each TFM | |
| for diff_file in comparisons/diff-*.md; do | |
| if [ -f "$diff_file" ]; then | |
| tfm=$(basename "$diff_file" .md | sed 's/diff-//') | |
| echo "### Results for $tfm" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| cat "$diff_file" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| done | |
| if [ ! -f comparisons/diff-*.md ]; then | |
| echo "⚠️ No comparison results available" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload Comparison Report | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: humanizer-bdn-comparison-${{ matrix.targetFramework }} | |
| path: comparisons/ | |
| retention-days: 7 |