Skip to content
Nightly Benchmarks

Nightly Benchmarks #102

Sign in to view logs

Nightly Benchmarks

Nightly Benchmarks #102

Workflow file for this run

.github/workflows/benchmarks.yml at ca68dae
name: Nightly Benchmarks
on:
schedule:
# Run every night at 02:00 UTC
- cron: '0 2 * * *'
workflow_dispatch:
permissions:
contents: write
pages: write
id-token: write
concurrency:
group: nightly-benchmarks
cancel-in-progress: false
env:
CARGO_TERM_COLOR: always
jobs:
# ---------------------------------------------------------------------------
# Matrix of per-crate bench binaries. Each bench runs in its own job so wall-
# clock is max(per-bench), not sum(per-bench). The previous monolithic job
# was hitting the GitHub-hosted 6h runner ceiling; splitting brings the full
# nightly down to roughly 50-55 minutes.
#
# Convention: matrix.name is the short slug (no `_bench` suffix). Output
# files use the slug (e.g. `ledger.txt`) so the report job can do a 1:1
# filename -> section mapping.
# ---------------------------------------------------------------------------
bench:
runs-on: ubuntu-latest
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
include:
- { name: storage, crate: dugite-storage, bench: storage_bench }
- { name: ledger, crate: dugite-ledger, bench: ledger_bench }
- { name: network, crate: dugite-network, bench: network_bench }
- { name: consensus, crate: dugite-consensus, bench: consensus_bench }
- { name: lsm, crate: dugite-lsm, bench: lsm_bench }
- { name: mempool, crate: dugite-mempool, bench: mempool_bench }
- { name: crypto, crate: dugite-crypto, bench: crypto_bench }
- { name: primitives, crate: dugite-primitives, bench: primitives_bench }
- { name: serialization, crate: dugite-serialization, bench: serialization_bench }
# lsm-stress runs `cargo test --features large-tests`, not `cargo
# bench`. Step branches on matrix.bench == "__stress__".
- { name: lsm-stress, crate: dugite-lsm, bench: __stress__ }
steps:
- uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# Per-bench Cargo cache: independent caches stop one bench's target dir
# thrashing another's. Mirrors the per-target cache shape used in
# fuzz.yml.
- name: Cache cargo registry and build
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-bench-${{ matrix.name }}-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-bench-${{ matrix.name }}-
# Per-bench Criterion baseline cache. No lockfile in the key so the
# `nightly` baseline persists across dep bumps and the run-to-run
# `change: +/-x%` line stays meaningful. Stress test produces no
# baseline.
- name: Cache Criterion baselines
if: matrix.bench != '__stress__'
uses: actions/cache@v5
with:
path: target/criterion
key: ${{ runner.os }}-criterion-${{ matrix.name }}
restore-keys: |
${{ runner.os }}-criterion-${{ matrix.name }}
# Strip cargo build chatter and ANSI escapes from captured bench output
# so the markdown report contains only criterion measurement lines.
# Full unfiltered output stays in *.full.txt for debugging.
- name: Define output filter
run: |
cat > /tmp/filter-bench-output.sh <<'EOF'
#!/usr/bin/env bash
sed -E 's/\x1B\[[0-9;]*[A-Za-z]//g' \
| grep -vE '^\s*(Updating|Downloading|Downloaded|Compiling|Fresh|Finished|Running|Locking|Blocking|Adding|Removing|warning:|note:)' \
| grep -vE '^\s*-->\s' \
| grep -vE '^\s*=\s*(help|note):' \
| grep -vE '^\s*\|' \
| cat -s
EOF
chmod +x /tmp/filter-bench-output.sh
# One `cargo bench` per matrix entry. `--save-baseline nightly` both
# produces the HTML report under target/criterion/<group>/ AND saves
# the run as the `nightly` baseline for the next run's comparison.
#
# Output is written line-buffered to `.full.txt` via `stdbuf -oL` and
# `tee`, so that even if this step is cancelled by `timeout-minutes`
# we still have a full transcript on disk. The filtered `.txt` is
# produced in a separate `if: always()` step below.
- name: Run benchmark
if: matrix.bench != '__stress__'
run: |
set +e
stdbuf -oL -eL cargo bench -p ${{ matrix.crate }} --bench ${{ matrix.bench }} --color never -- --save-baseline nightly --color never 2>&1 \
| stdbuf -oL tee ${{ matrix.name }}.full.txt
exit 0
# The LSM mainnet-scale tests are `large-tests` integration tests, not
# Criterion benches. They produce no target/criterion/ tree.
- name: Run LSM mainnet-scale stress tests
if: matrix.bench == '__stress__'
run: |
set +e
stdbuf -oL -eL cargo test -p dugite-lsm --features large-tests --release --color never -- mainnet_scale --nocapture 2>&1 \
| stdbuf -oL tee ${{ matrix.name }}.full.txt
exit 0
# Filter the full transcript into the markdown-ready `.txt`. Runs even
# if the bench step was cancelled or failed, so the aggregate report
# always has measurement lines for whatever did complete.
- name: Filter bench output
if: always()
run: |
if [ -f ${{ matrix.name }}.full.txt ]; then
/tmp/filter-bench-output.sh < ${{ matrix.name }}.full.txt > ${{ matrix.name }}.txt || true
else
printf 'No output captured. The bench step did not produce a full.txt transcript.\n' > ${{ matrix.name }}.txt
fi
echo "--- ${{ matrix.name }}.txt size: $(wc -c < ${{ matrix.name }}.txt) bytes"
- name: Upload bench artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: bench-${{ matrix.name }}
path: |
${{ matrix.name }}.txt
${{ matrix.name }}.full.txt
target/criterion/
if-no-files-found: warn
retention-days: 30
# ---------------------------------------------------------------------------
# Aggregate report. Runs after all matrix entries (always(), so a partial
# bench failure still produces a report for the suites that did pass).
# Only commits to main and publishes Pages from the main branch.
# ---------------------------------------------------------------------------
report:
needs: bench
if: always() && github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
- name: Download all bench artifacts
uses: actions/download-artifact@v7
with:
pattern: bench-*
path: ./_artifacts
# Reassemble the per-job criterion trees into one combined tree at
# ./target/criterion/ so the Pages publish step sees every suite.
# Each bench-<name>/target/criterion/* contains disjoint group names.
- name: Reassemble criterion tree and bench logs
run: |
mkdir -p target/criterion
for dir in _artifacts/bench-*; do
name=${dir#_artifacts/bench-}
[ -f "$dir/${name}.txt" ] && cp "$dir/${name}.txt" ./
[ -f "$dir/${name}.full.txt" ] && cp "$dir/${name}.full.txt" ./
if [ -d "$dir/target/criterion" ]; then
cp -R "$dir/target/criterion/." target/criterion/
fi
done
ls -la *.txt 2>/dev/null || true
ls -la target/criterion/ 2>/dev/null || true
- name: Generate benchmark report
run: |
DATE=$(date -u +%Y-%m-%d)
SHORT_SHA=$(git rev-parse --short HEAD)
mkdir -p benches
REPORT="benches/${DATE}-nightly.md"
emit_section() {
local title="$1" file="$2"
local full="${file%.txt}.full.txt"
printf '## %s\n\n' "$title" >> "$REPORT"
if [ -f "$file" ] && [ -s "$file" ]; then
printf '<details>\n<summary>Raw measurements</summary>\n\n```\n' >> "$REPORT"
cat "$file" >> "$REPORT"
printf '\n```\n\n</details>\n\n' >> "$REPORT"
elif [ -f "$full" ] && [ -s "$full" ]; then
# Filtered output is empty (likely cancelled by job timeout);
# surface the tail of the unfiltered transcript so the report
# at least shows what the bench was working on when it died.
printf '_Filtered measurements were not captured — the job was likely cancelled by `timeout-minutes` before completing. Tail of the unfiltered transcript:_\n\n' >> "$REPORT"
printf '<details>\n<summary>Tail of full transcript</summary>\n\n```\n' >> "$REPORT"
tail -n 80 "$full" >> "$REPORT"
printf '\n```\n\n</details>\n\n' >> "$REPORT"
else
printf '_No results captured. The bench step produced no output (likely a build failure — check the workflow run log)._\n\n' >> "$REPORT"
fi
}
{
printf '# Nightly Benchmark Results — %s\n\n' "$DATE"
printf 'Captured on **%s** from commit [`%s`](https://github.com/MichaelJFazio/dugite/commit/%s) on `main` (GitHub Actions `ubuntu-latest`).\n\n' \
"$DATE" "$SHORT_SHA" "$SHORT_SHA"
printf '> **Interactive reports**, including per-benchmark detail pages and historical trend lines, are published at <https://michaeljfazio.github.io/dugite/benchmarks/>. Each section below also links directly to its interactive report.\n\n'
printf '> The collapsed _Raw measurements_ blocks contain the filtered `cargo bench` output (cargo build chatter and ANSI escapes are stripped). Full unfiltered logs are uploaded as the `benchmark-results-${{ github.run_number }}` workflow artifact.\n\n'
printf -- '---\n\n'
} > "$REPORT"
emit_section "Storage" storage.txt
emit_section "Ledger (UTxO)" ledger.txt
emit_section "Network" network.txt
emit_section "Consensus" consensus.txt
emit_section "LSM" lsm.txt
emit_section "Mempool" mempool.txt
emit_section "Crypto" crypto.txt
emit_section "Primitives" primitives.txt
emit_section "Serialization" serialization.txt
emit_section "LSM stress tests" lsm-stress.txt
- name: Upload aggregated benchmark report
uses: actions/upload-artifact@v7
with:
name: benchmark-results-${{ github.run_number }}
path: |
benches/*-nightly.md
*.full.txt
retention-days: 90
- name: Copy results to docs and commit
run: |
DATE=$(date -u +%Y-%m-%d)
LATEST="benches/${DATE}-nightly.md"
if [ -f "$LATEST" ]; then
cp "$LATEST" docs/src/reference/benchmarks.md
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git add "benches/${DATE}-nightly.md" docs/src/reference/benchmarks.md
git commit -m "Update nightly benchmark results (${DATE})" || echo "No changes to commit"
git pull --rebase origin main || true
git push || echo "Push failed — results saved as artifact"
fi
- name: Store Criterion HTML reports
uses: actions/upload-artifact@v7
with:
name: criterion-reports-${{ github.run_number }}
path: target/criterion/
retention-days: 30
- name: Publish Criterion HTML to GitHub Pages
run: |
if [ -d "target/criterion" ]; then
mkdir -p mdbook && curl -sSL https://github.com/rust-lang/mdBook/releases/download/v0.4.40/mdbook-v0.4.40-x86_64-unknown-linux-gnu.tar.gz | tar -xz -C mdbook
export PATH="$(pwd)/mdbook:$PATH"
mdbook build docs || true
mkdir -p docs/book/benchmarks
cp -r target/criterion/* docs/book/benchmarks/ 2>/dev/null || true
cat > docs/book/benchmarks/index.html <<BENCHHTML
<!DOCTYPE html>
<html><head><title>Dugite Benchmarks</title></head>
<body>
<h1>Dugite Criterion Benchmark Reports</h1>
<p>Interactive benchmark reports from the nightly CI run.</p>
<ul>
BENCHHTML
for dir in docs/book/benchmarks/*/; do
name=$(basename "$dir")
if [ -f "$dir/report/index.html" ]; then
echo "<li><a href=\"$name/report/index.html\">$name</a></li>" >> docs/book/benchmarks/index.html
fi
done
echo "</ul></body></html>" >> docs/book/benchmarks/index.html
fi
- name: Deploy to GitHub Pages
if: always()
uses: peaceiris/actions-gh-pages@v4
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./docs/book
destination_dir: .
keep_files: false
⚡