feat(benchmark-tests): enhance hydration benchmarks with batch processing and result tracking

- Increased default test timeout from 5000ms to 10000ms for better stability.
- Implemented batch processing in hydration tests to improve performance measurement.
- Introduced a results map to collect and store hydration durations for each tag.
- Updated test configurations to accommodate new batch size and timeout settings.
- Refactored result writing logic into a dedicated module for better organization.
- Enhanced the test run logic to handle batches of elements and track hydration performance more effectively.

Author: deleonio

packages/components/src/components/toaster/toaster.tsx

@@ -38,7 +38,7 @@ export class ToasterService {
 			this.toastContainerElement = undefined;
 			element.remove();
 		} else {
-			Log.warn('Toaster service is already disposed.', { forceLog: true });
+			Log.warn('Toaster service is already disposed.');
 		}
 	}
 

packages/components/src/schema/props/color.ts

@@ -108,7 +108,7 @@ export const handleColorChange = (value: unknown): ColorPair => {
 			break;
 		}
 		case null:
-			Log.warn(`_color was empty or invalid (${JSON.stringify(value)})`, { forceLog: true });
+			Log.warn(`_color was empty or invalid (${JSON.stringify(value)})`);
 			colorContrast = createContrastColorPair({
 				background: '#000',
 				foreground: '#000',

packages/components/src/utils/unique-nav-labels.ts

@@ -9,7 +9,7 @@ const UNIQUE_LABELS: Set<string> = new Set();
 
 export function addNavLabel(ariaLabel: string): void {
 	if (UNIQUE_LABELS.has(ariaLabel)) {
-		Log.error(`There already is a nav element with the label "${ariaLabel}"`, { forceLog: true });
+		Log.warn(`There already is a nav element with the label "${ariaLabel}"`);
 	} else {
 		UNIQUE_LABELS.add(ariaLabel);
 	}

packages/tools/benchmark-tests/playwright.config.js

@@ -1,11 +1,11 @@
 import { defineConfig, devices } from '@playwright/test';
 
-const TEST_TIMEOUT = parseInt(process.env.TEST_TIMEOUT || '5000', 10);
+const TEST_TIMEOUT = parseInt(process.env.TEST_TIMEOUT || '10000', 10);
 
 export default defineConfig({
 	testDir: 'tests',
 	testMatch: '*.playwright.ts',
-	timeout: TEST_TIMEOUT,
+	timeout: 2 * TEST_TIMEOUT,
 	use: {
 		headless: true,
 		viewport: { width: 800, height: 600 },

packages/tools/benchmark-tests/tests/hydration.playwright.ts

@@ -1,16 +1,21 @@
 import { test } from '@playwright/test';
-import { TAGS, TEST_URL } from './lib/config';
-import { runBenchmark, writeResultFile } from './lib/test';
+import { createResultsMap, TAGS, TEST_URL } from './lib/config';
 import type { Params } from './lib/types';
+import { runBenchmark } from './lib/browser';
+import { writeResultFile } from './lib/after';
+
+const RESULTS = createResultsMap();
 
 test.describe('Hydration Benchmark', () => {
 	test.beforeEach(async ({ page }) => {
 		await page.goto(TEST_URL);
 	});
 
 	for (const tag of TAGS) {
-		test(`${tag}`, async ({ page }) => await runBenchmark(tag, (fn: any, params: Params) => page.evaluate(fn, params)));
+		test(`${tag}`, async ({ page }) => await runBenchmark(tag, (fn: any, params: Params) => page.evaluate(fn, params), RESULTS));
 	}
 
-	test.afterAll(writeResultFile);
+	test.afterAll(() => {
+		writeResultFile(RESULTS);
+	});
 });

packages/tools/benchmark-tests/tests/hydration.webdriver.ts

@@ -1,15 +1,20 @@
-import { TAGS, TEST_URL } from './lib/config';
-import { runBenchmark, writeResultFile } from './lib/test';
+import { writeResultFile } from './lib/after';
+import { runBenchmark } from './lib/browser';
+import { createResultsMap, TAGS, TEST_URL } from './lib/config';
 import type { Params } from './lib/types';
 
+const RESULTS = createResultsMap();
+
 describe('Hydration Benchmark', () => {
 	before(async () => {
 		await browser.url(TEST_URL);
 	});
 
 	for (const tag of TAGS) {
-		it(`${tag}`, async () => await runBenchmark(tag, (fn: any, params: Params) => browser.execute(fn, params)));
+		it(`${tag}`, async () => await runBenchmark(tag, (fn: any, params: Params) => browser.execute(fn, params), RESULTS));
 	}
 
-	after(writeResultFile);
+	after(() => {
+		writeResultFile(RESULTS);
+	});
 });

packages/tools/benchmark-tests/tests/lib/after.ts

@@ -0,0 +1,34 @@
+import { writeFileSync } from 'fs';
+import type { TagType } from './types';
+
+export function writeResultFile(results: Map<TagType, number[]>) {
+	function percentile(sorted: number[], p: number): number {
+		const i = Math.floor(sorted.length * p);
+		return sorted[i] ?? sorted[sorted.length - 1];
+	}
+
+	function stddev(arr: number[]) {
+		const mean = arr.reduce((a, b) => a + b, 0) / arr.length;
+		const squared = arr.map((x) => (x - mean) ** 2);
+		return Math.sqrt(squared.reduce((a, b) => a + b, 0) / arr.length);
+	}
+
+	const finalResults = Array.from(results.entries())
+		.filter(([_, values]) => values.length > 0)
+		.map(([tag, values]) => {
+			values.sort((a, b) => a - b);
+			const mid = Math.floor(values.length / 2);
+			return {
+				name: tag,
+				unit: 'ms',
+				value: values[mid],
+				p95: percentile(values, 0.95),
+				p99: percentile(values, 0.99),
+				min: values[0],
+				max: values[values.length - 1],
+				stddev: stddev(values),
+			};
+		});
+
+	writeFileSync('benchmark-result.json', JSON.stringify(finalResults, null, 2));
+}

packages/tools/benchmark-tests/tests/lib/browser.ts

@@ -0,0 +1,18 @@
+import { TEST_BATCH_SIZE, TEST_ITERATIONS, TEST_TIMEOUT } from './config';
+import { testRun } from './test';
+import type { TagType } from './types';
+
+export async function runBenchmark(tag: TagType, execFn: any, results: Map<TagType, number[]>) {
+	const durations: number[] = await execFn(testRun, {
+		batchSize: TEST_BATCH_SIZE,
+		iterations: TEST_ITERATIONS,
+		tag,
+		timeout: TEST_TIMEOUT,
+	});
+
+	console.log(`Hydration durations for ${tag}:`, durations);
+	/**
+	 * Cut warmup iterations from the results.
+	 */
+	results.set(tag, durations.splice(1, durations.length - 1));
+}

packages/tools/benchmark-tests/tests/lib/config.ts

@@ -1,6 +1,8 @@
-export const TEST_ITERATIONS = parseInt(process.env.TEST_ITERATIONS || '25', 10);
-export const TEST_TIMEOUT = parseInt(process.env.TEST_TIMEOUT || '5000', 10);
-// export const TEST_BATCH_SIZE = parseInt(process.env.TEST_BATCH_SIZE || '10', 10);
+import type { TagType } from './types';
+
+export const TEST_ITERATIONS = parseInt(process.env.TEST_ITERATIONS || '50', 10);
+export const TEST_BATCH_SIZE = parseInt(process.env.TEST_BATCH_SIZE || '100', 10);
+export const TEST_TIMEOUT = parseInt(process.env.TEST_TIMEOUT || '10000', 10);
 export const TEST_URL = 'http://localhost:3000/test-page.html';
 
 export const TAGS = [
@@ -51,3 +53,7 @@ export const TAGS = [
 	'kol-tree-item',
 	'kol-version',
 ] as const;
+
+export const createResultsMap = (): Map<TagType, number[]> => {
+	return new Map<TagType, number[]>(TAGS.map((tag) => [tag, []]));
+};

packages/tools/benchmark-tests/tests/lib/test.ts

@@ -1,25 +1,33 @@
-import { writeFileSync } from 'fs';
-import { TEST_ITERATIONS, TEST_TIMEOUT } from './config';
-import type { Measure, Params, TagType } from './types';
+import type { Measure, Params } from './types';
 
-async function testRun({ iterations, tag, timeout }: Params): Promise<number[]> {
+export async function testRun({ batchSize, iterations, tag, timeout }: Params): Promise<number[]> {
 	return new Promise(async (resolve) => {
-		const testResults = new Map<HTMLElement, Measure>();
-		const webComponents = new Set<HTMLElement>();
-		const batches = [];
+		const testResults: Map<Set<HTMLElement>, Measure> = new Map();
+		const batches: Set<Set<HTMLElement>> = new Set();
+		let currentBatch: Set<HTMLElement> = new Set();
+		let batchCounter = 0;
 
 		window.gc?.();
 		await customElements.whenDefined(tag);
 
+		// Fallback for when no elements are hydrated
+		setTimeout(() => {
+			console.warn('⚠️ Timeout: hydration did not finish in time – returning partial or empty results.');
+			returnDurations();
+		}, timeout);
+
 		function startNextHydration() {
-			if (webComponents.size > 0) {
-				const el: HTMLElement = webComponents.values()?.next()?.value!;
-				performance.mark(`mark-append-${el.getAttribute('data-iteration')}`);
-				testResults.set(el, {
+			if (batches.size > 0) {
+				batchCounter++;
+				currentBatch = batches.values()?.next()?.value!;
+				performance.mark(`mark-append-${batchCounter}`);
+				testResults.set(currentBatch, {
 					hydrated: null,
 					themed: null,
 				});
-				document.body.appendChild(el);
+				for (const el of currentBatch) {
+					document.body.appendChild(el);
+				}
 			} else {
 				returnDurations();
 			}
@@ -36,8 +44,12 @@ async function testRun({ iterations, tag, timeout }: Params): Promise<number[]>
 			resolve(durations);
 		}
 
+		function removeBatch(batch: Set<HTMLElement>) {
+			batches.delete(batch);
+		}
+
 		function removeElement(el: HTMLElement) {
-			webComponents.delete(el);
+			currentBatch.delete(el);
 			try {
 				el.remove();
 			} catch {}
@@ -46,89 +58,37 @@ async function testRun({ iterations, tag, timeout }: Params): Promise<number[]>
 		const observer = new MutationObserver((mutations) => {
 			for (const mutation of mutations) {
 				const el = mutation.target as HTMLElement;
-				if (!webComponents.has(el) && !el.isConnected) continue;
-
-				const measure = testResults.get(el);
-				if (!measure) continue;
+				if (!currentBatch.has(el) && !el.isConnected) continue;
 
-				const id = el.getAttribute('data-iteration');
+				if (el.classList.contains('hydrated')) {
+					removeElement(el);
+					if (currentBatch.size === 0) {
+						performance.mark(`mark-hydrated-${batchCounter}`);
+						performance.measure(`hydrated-${batchCounter}`, `mark-append-${batchCounter}`, `mark-hydrated-${batchCounter}`);
 
-				if (!measure.hydrated && el.classList.contains('hydrated')) {
-					performance.mark(`mark-hydrated-${id}`);
-					performance.measure(`hydrated-${id}`, `mark-append-${id}`, `mark-hydrated-${id}`);
-					measure.hydrated = performance.getEntriesByName(`hydrated-${id}`).pop()?.duration!;
-					// } else if (measure.hydrated && el.hasAttribute('data-themed')) {
-					// 	performance.mark(`mark-themed-${id}`);
-					// 	performance.measure(`themed-${id}`, `mark-append-${id}`, `mark-themed-${id}`);
-					// 	measure.themed = performance.getEntriesByName(`themed-${id}`).pop()?.duration!;
+						const measure = testResults.get(currentBatch)!;
+						measure.hydrated = performance.getEntriesByName(`hydrated-${batchCounter}`).pop()?.duration!;
 
-					removeElement(el);
-					startNextHydration();
+						removeBatch(currentBatch);
+						setTimeout(startNextHydration);
+					}
 				}
 			}
 		});
 
 		for (let i = 0; i <= iterations; i++) {
-			const el = document.createElement(tag);
-			el.setAttribute('data-iteration', i.toString());
-			webComponents.add(el);
-			observer.observe(el, {
-				attributes: true,
-				attributeFilter: ['class', 'data-themed'],
-			});
+			const batch = new Set<HTMLElement>();
+			for (let j = 0; j < batchSize; j++) {
+				const el = document.createElement(tag);
+				batch.add(el);
+				observer.observe(el, {
+					attributes: true,
+					attributeFilter: ['class', 'data-themed'],
+				});
+			}
+			batches.add(batch);
 		}
 
-		// Fallback for when no elements are hydrated
-		setTimeout(returnDurations, timeout);
-
 		startNextHydration();
 	});
 }
-
-const results = new Map<TagType, number[]>();
-export function writeResultFile() {
-	function percentile(sorted, p) {
-		const i = Math.floor(sorted.length * p);
-		return sorted[i] ?? sorted[sorted.length - 1];
-	}
-
-	function stddev(arr) {
-		const mean = arr.reduce((a, b) => a + b, 0) / arr.length;
-		const squared = arr.map((x) => (x - mean) ** 2);
-		return Math.sqrt(squared.reduce((a, b) => a + b, 0) / arr.length);
-	}
-
-	const finalResults = Array.from(results.entries())
-		.filter(([_, values]) => values.length > 0)
-		.map(([tag, values]) => {
-			values.sort((a, b) => a - b);
-			const mid = Math.floor(values.length / 2);
-			return {
-				name: tag,
-				unit: 'ms',
-				value: values[mid],
-				p95: percentile(values, 0.95),
-				p99: percentile(values, 0.99),
-				min: values[0],
-				max: values[values.length - 1],
-				stddev: stddev(values),
-			};
-		});
-
-	writeFileSync('benchmark-result.json', JSON.stringify(finalResults, null, 2));
-}
-
-export async function runBenchmark(tag: TagType, execFn: any) {
-	const durations: number[] = await execFn(testRun, {
-		iterations: TEST_ITERATIONS,
-		tag,
-		timeout: TEST_TIMEOUT,
-	});
-
-	console.log(`Hydration durations for ${tag}:`, durations);
-	/**
-	 * The network request durations are removed from the test results
-	 * to focus on the hydration performance of the web components itself.
-	 */
-	results.set(tag, durations.splice(1, durations.length - 1));
-}