Nightly Benchmarks #88

Workflow file for this run

.github/workflows/benchmarks.yml at fb50038

	name: Nightly Benchmarks

	on:
	schedule:
	# Run every night at 02:00 UTC
	- cron: '0 2 * * *'
	workflow_dispatch:

	permissions:
	contents: write
	pages: write
	id-token: write

	concurrency:
	group: nightly-benchmarks
	cancel-in-progress: false

	env:
	CARGO_TERM_COLOR: always

	jobs:
	# ---------------------------------------------------------------------------
	# Matrix of per-crate bench binaries. Each bench runs in its own job so wall-
	# clock is max(per-bench), not sum(per-bench). The previous monolithic job
	# was hitting the GitHub-hosted 6h runner ceiling; splitting brings the full
	# nightly down to roughly 50-55 minutes.
	#
	# Convention: matrix.name is the short slug (no `_bench` suffix). Output
	# files use the slug (e.g. `ledger.txt`) so the report job can do a 1:1
	# filename -> section mapping.
	# ---------------------------------------------------------------------------
	bench:
	runs-on: ubuntu-latest
	timeout-minutes: 90
	strategy:
	fail-fast: false
	matrix:
	include:
	- { name: storage, crate: dugite-storage, bench: storage_bench }
	- { name: ledger, crate: dugite-ledger, bench: ledger_bench }
	- { name: network, crate: dugite-network, bench: network_bench }
	- { name: consensus, crate: dugite-consensus, bench: consensus_bench }
	- { name: lsm, crate: dugite-lsm, bench: lsm_bench }
	- { name: mempool, crate: dugite-mempool, bench: mempool_bench }
	- { name: crypto, crate: dugite-crypto, bench: crypto_bench }
	- { name: primitives, crate: dugite-primitives, bench: primitives_bench }
	- { name: serialization, crate: dugite-serialization, bench: serialization_bench }
	# lsm-stress runs `cargo test --features large-tests`, not `cargo
	# bench`. Step branches on matrix.bench == "__stress__".
	- { name: lsm-stress, crate: dugite-lsm, bench: __stress__ }
	steps:
	- uses: actions/checkout@v6

	- name: Install Rust toolchain
	uses: dtolnay/rust-toolchain@stable

	# Per-bench Cargo cache: independent caches stop one bench's target dir
	# thrashing another's. Mirrors the per-target cache shape used in
	# fuzz.yml.
	- name: Cache cargo registry and build
	uses: actions/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-bench-${{ matrix.name }}-${{ hashFiles('**/Cargo.lock') }}
	restore-keys: \|
	${{ runner.os }}-bench-${{ matrix.name }}-

	# Per-bench Criterion baseline cache. No lockfile in the key so the
	# `nightly` baseline persists across dep bumps and the run-to-run
	# `change: +/-x%` line stays meaningful. Stress test produces no
	# baseline.
	- name: Cache Criterion baselines
	if: matrix.bench != '__stress__'
	uses: actions/cache@v5
	with:
	path: target/criterion
	key: ${{ runner.os }}-criterion-${{ matrix.name }}
	restore-keys: \|
	${{ runner.os }}-criterion-${{ matrix.name }}

	# Strip cargo build chatter and ANSI escapes from captured bench output
	# so the markdown report contains only criterion measurement lines.
	# Full unfiltered output stays in *.full.txt for debugging.
	- name: Define output filter
	run: \|
	cat > /tmp/filter-bench-output.sh <<'EOF'
	#!/usr/bin/env bash
	sed -E 's/\x1B\[[0-9;]*[A-Za-z]//g' \
	\| grep -vE '^\s*(Updating\|Downloading\|Downloaded\|Compiling\|Fresh\|Finished\|Running\|Locking\|Blocking\|Adding\|Removing\|warning:\|note:)' \
	\| grep -vE '^\s*-->\s' \
	\| grep -vE '^\s=\s(help\|note):' \
	\| grep -vE '^\s*\\|' \
	\| cat -s
	EOF
	chmod +x /tmp/filter-bench-output.sh

	# One `cargo bench` per matrix entry. `--save-baseline nightly` both
	# produces the HTML report under target/criterion/<group>/ AND saves
	# the run as the `nightly` baseline for the next run's comparison.
	#
	# Output is written line-buffered to `.full.txt` via `stdbuf -oL` and
	# `tee`, so that even if this step is cancelled by `timeout-minutes`
	# we still have a full transcript on disk. The filtered `.txt` is
	# produced in a separate `if: always()` step below.
	- name: Run benchmark
	if: matrix.bench != '__stress__'
	run: \|
	set +e
	stdbuf -oL -eL cargo bench -p ${{ matrix.crate }} --bench ${{ matrix.bench }} --color never -- --save-baseline nightly --color never 2>&1 \
	\| stdbuf -oL tee ${{ matrix.name }}.full.txt
	exit 0

	# The LSM mainnet-scale tests are `large-tests` integration tests, not
	# Criterion benches. They produce no target/criterion/ tree.
	- name: Run LSM mainnet-scale stress tests
	if: matrix.bench == '__stress__'
	run: \|
	set +e
	stdbuf -oL -eL cargo test -p dugite-lsm --features large-tests --release --color never -- mainnet_scale --nocapture 2>&1 \
	\| stdbuf -oL tee ${{ matrix.name }}.full.txt
	exit 0

	# Filter the full transcript into the markdown-ready `.txt`. Runs even
	# if the bench step was cancelled or failed, so the aggregate report
	# always has measurement lines for whatever did complete.
	- name: Filter bench output
	if: always()
	run: \|
	if [ -f ${{ matrix.name }}.full.txt ]; then
	/tmp/filter-bench-output.sh < ${{ matrix.name }}.full.txt > ${{ matrix.name }}.txt \|\| true
	else
	printf 'No output captured. The bench step did not produce a full.txt transcript.\n' > ${{ matrix.name }}.txt
	fi
	echo "--- ${{ matrix.name }}.txt size: $(wc -c < ${{ matrix.name }}.txt) bytes"

	- name: Upload bench artifact
	if: always()
	uses: actions/upload-artifact@v7
	with:
	name: bench-${{ matrix.name }}
	path: \|
	${{ matrix.name }}.txt
	${{ matrix.name }}.full.txt
	target/criterion/
	if-no-files-found: warn
	retention-days: 30

	# ---------------------------------------------------------------------------
	# Aggregate report. Runs after all matrix entries (always(), so a partial
	# bench failure still produces a report for the suites that did pass).
	# Only commits to main and publishes Pages from the main branch.
	# ---------------------------------------------------------------------------
	report:
	needs: bench
	if: always() && github.ref == 'refs/heads/main'
	runs-on: ubuntu-latest
	timeout-minutes: 30
	steps:
	- uses: actions/checkout@v6

	- name: Download all bench artifacts
	uses: actions/download-artifact@v7
	with:
	pattern: bench-*
	path: ./_artifacts

	# Reassemble the per-job criterion trees into one combined tree at
	# ./target/criterion/ so the Pages publish step sees every suite.
	# Each bench-<name>/target/criterion/* contains disjoint group names.
	- name: Reassemble criterion tree and bench logs
	run: \|
	mkdir -p target/criterion
	for dir in _artifacts/bench-*; do
	name=${dir#_artifacts/bench-}
	[ -f "$dir/${name}.txt" ] && cp "$dir/${name}.txt" ./
	[ -f "$dir/${name}.full.txt" ] && cp "$dir/${name}.full.txt" ./
	if [ -d "$dir/target/criterion" ]; then
	cp -R "$dir/target/criterion/." target/criterion/
	fi
	done
	ls -la *.txt 2>/dev/null \|\| true
	ls -la target/criterion/ 2>/dev/null \|\| true

	- name: Generate benchmark report
	run: \|
	DATE=$(date -u +%Y-%m-%d)
	SHORT_SHA=$(git rev-parse --short HEAD)
	mkdir -p benches
	REPORT="benches/${DATE}-nightly.md"

	emit_section() {
	local title="$1" file="$2"
	local full="${file%.txt}.full.txt"
	printf '## %s\n\n' "$title" >> "$REPORT"
	if [ -f "$file" ] && [ -s "$file" ]; then
	printf '<details>\n<summary>Raw measurements</summary>\n\n```\n' >> "$REPORT"
	cat "$file" >> "$REPORT"
	printf '\n```\n\n</details>\n\n' >> "$REPORT"
	elif [ -f "$full" ] && [ -s "$full" ]; then
	# Filtered output is empty (likely cancelled by job timeout);
	# surface the tail of the unfiltered transcript so the report
	# at least shows what the bench was working on when it died.
	printf '_Filtered measurements were not captured — the job was likely cancelled by `timeout-minutes` before completing. Tail of the unfiltered transcript:_\n\n' >> "$REPORT"
	printf '<details>\n<summary>Tail of full transcript</summary>\n\n```\n' >> "$REPORT"
	tail -n 80 "$full" >> "$REPORT"
	printf '\n```\n\n</details>\n\n' >> "$REPORT"
	else
	printf '_No results captured. The bench step produced no output (likely a build failure — check the workflow run log)._\n\n' >> "$REPORT"
	fi
	}

	{
	printf '# Nightly Benchmark Results — %s\n\n' "$DATE"
	printf 'Captured on %s from commit [`%s`](https://github.com/MichaelJFazio/dugite/commit/%s) on `main` (GitHub Actions `ubuntu-latest`).\n\n' \
	"$DATE" "$SHORT_SHA" "$SHORT_SHA"
	printf '> Interactive reports, including per-benchmark detail pages and historical trend lines, are published at <https://michaeljfazio.github.io/dugite/benchmarks/>. Each section below also links directly to its interactive report.\n\n'
	printf '> The collapsed _Raw measurements_ blocks contain the filtered `cargo bench` output (cargo build chatter and ANSI escapes are stripped). Full unfiltered logs are uploaded as the `benchmark-results-${{ github.run_number }}` workflow artifact.\n\n'
	printf -- '---\n\n'
	} > "$REPORT"

	emit_section "Storage" storage.txt
	emit_section "Ledger (UTxO)" ledger.txt
	emit_section "Network" network.txt
	emit_section "Consensus" consensus.txt
	emit_section "LSM" lsm.txt
	emit_section "Mempool" mempool.txt
	emit_section "Crypto" crypto.txt
	emit_section "Primitives" primitives.txt
	emit_section "Serialization" serialization.txt
	emit_section "LSM stress tests" lsm-stress.txt

	- name: Upload aggregated benchmark report
	uses: actions/upload-artifact@v7
	with:
	name: benchmark-results-${{ github.run_number }}
	path: \|
	benches/*-nightly.md
	*.full.txt
	retention-days: 90

	- name: Copy results to docs and commit
	run: \|
	DATE=$(date -u +%Y-%m-%d)
	LATEST="benches/${DATE}-nightly.md"
	if [ -f "$LATEST" ]; then
	cp "$LATEST" docs/src/reference/benchmarks.md
	git config user.name "github-actions[bot]"
	git config user.email "github-actions[bot]@users.noreply.github.com"
	git add "benches/${DATE}-nightly.md" docs/src/reference/benchmarks.md
	git commit -m "Update nightly benchmark results (${DATE})" \|\| echo "No changes to commit"
	git pull --rebase origin main \|\| true
	git push \|\| echo "Push failed — results saved as artifact"
	fi

	- name: Store Criterion HTML reports
	uses: actions/upload-artifact@v7
	with:
	name: criterion-reports-${{ github.run_number }}
	path: target/criterion/
	retention-days: 30

	- name: Publish Criterion HTML to GitHub Pages
	run: \|
	if [ -d "target/criterion" ]; then
	mkdir -p mdbook && curl -sSL https://github.com/rust-lang/mdBook/releases/download/v0.4.40/mdbook-v0.4.40-x86_64-unknown-linux-gnu.tar.gz \| tar -xz -C mdbook
	export PATH="$(pwd)/mdbook:$PATH"
	mdbook build docs \|\| true
	mkdir -p docs/book/benchmarks
	cp -r target/criterion/* docs/book/benchmarks/ 2>/dev/null \|\| true
	cat > docs/book/benchmarks/index.html <<BENCHHTML
	<!DOCTYPE html>
	<html><head><title>Dugite Benchmarks</title></head>
	<body>
	<h1>Dugite Criterion Benchmark Reports</h1>
	<p>Interactive benchmark reports from the nightly CI run.</p>
	<ul>
	BENCHHTML
	for dir in docs/book/benchmarks/*/; do
	name=$(basename "$dir")
	if [ -f "$dir/report/index.html" ]; then
	echo "<li><a href=\"$name/report/index.html\">$name</a></li>" >> docs/book/benchmarks/index.html
	fi
	done
	echo "</ul></body></html>" >> docs/book/benchmarks/index.html
	fi

	- name: Deploy to GitHub Pages
	if: always()
	uses: peaceiris/actions-gh-pages@v4
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	publish_dir: ./docs/book
	destination_dir: .
	keep_files: false

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Nightly Benchmarks #88

Workflow file

Nightly Benchmarks #88

Uh oh!

Workflow file for this run