Add 'describeAgentDocsPerCheck' and CI-friendly reporting

Author: dacharyc

README.md

@@ -124,32 +124,90 @@ const result = await check.run(ctx);
 
 ## Test helpers
 
-afdocs includes vitest helpers so you can add agent-friendliness checks to your docs site's test suite.
+afdocs includes vitest helpers so you can add agent-friendliness checks to your docs site's CI pipeline.
 
-### Config-driven
+### Setup
 
-Create `agent-docs.config.yml`:
+Install afdocs and vitest:
+
+```bash
+npm install -D afdocs vitest
+```
+
+Create `agent-docs.config.yml` in your project root (or a `tests/` subdirectory):
+
+```yaml
+url: https://docs.example.com
+```
+
+Create a test file:
+
+```ts
+import { describeAgentDocsPerCheck } from 'afdocs/helpers';
+
+describeAgentDocsPerCheck();
+```
+
+Run it:
+
+```bash
+npx vitest run agent-docs.test.ts
+```
+
+Each check appears as its own test in the output, so you can see exactly what passed, warned, failed, or was skipped:
+
+```
+ ✓ Agent-Friendly Documentation > llms-txt-exists
+ ✓ Agent-Friendly Documentation > llms-txt-valid
+ ✓ Agent-Friendly Documentation > llms-txt-size
+ × Agent-Friendly Documentation > markdown-url-support
+ ↓ Agent-Friendly Documentation > page-size-markdown
+```
+
+Checks that fail cause the test to fail. Checks that warn still pass (they're informational). Checks skipped due to unmet dependencies or config filtering show as skipped.
+
+### Running a subset of checks
+
+If your platform doesn't support certain checks (for example, you can't serve markdown), you can limit which checks run via the config:
 
 ```yaml
 url: https://docs.example.com
 checks:
   - llms-txt-exists
   - llms-txt-valid
   - llms-txt-size
+  - http-status-codes
+  - auth-gate-detection
 ```
 
-Then in your test file:
+Only the listed checks will run. The rest show as skipped in the test output.
+
+### Config resolution
+
+The helpers look for `agent-docs.config.yml` (or `.yaml`) starting from `process.cwd()` and walking up the directory tree, so the config works whether your test file is at the project root or in a subdirectory. You can also pass an explicit directory:
+
+```ts
+describeAgentDocsPerCheck(__dirname);
+```
+
+### Timeouts
+
+The helpers set a 120-second timeout on the check run automatically. No vitest timeout configuration is needed.
+
+### Summary helper
+
+If you don't need per-check granularity, `describeAgentDocs` provides a simpler two-test suite (one to run checks, one to assert no failures):
 
 ```ts
 import { describeAgentDocs } from 'afdocs/helpers';
 
 describeAgentDocs();
 ```
 
-This reads the config and generates one test assertion covering all specified checks.
-
 ### Direct imports
 
+For full control, use the programmatic API directly:
+
 ```ts
 import { createContext, getCheck } from 'afdocs';
 import { describe, it, expect } from 'vitest';

src/helpers/config.ts

@@ -1,26 +1,37 @@
 import { readFile } from 'node:fs/promises';
-import { resolve } from 'node:path';
+import { dirname, resolve } from 'node:path';
 import { parse as parseYaml } from 'yaml';
 import type { AgentDocsConfig } from '../types.js';
 
 const CONFIG_FILENAMES = ['agent-docs.config.yml', 'agent-docs.config.yaml'];
 
+/**
+ * Search for an agent-docs config file starting from `dir` and walking up
+ * to the filesystem root (like eslint, prettier, etc.).
+ * If `dir` is omitted, starts from `process.cwd()`.
+ */
 export async function loadConfig(dir?: string): Promise<AgentDocsConfig> {
-  const searchDir = dir ?? process.cwd();
+  let searchDir = resolve(dir ?? process.cwd());
 
-  for (const filename of CONFIG_FILENAMES) {
-    const filepath = resolve(searchDir, filename);
-    try {
-      const content = await readFile(filepath, 'utf-8');
-      const parsed = parseYaml(content) as AgentDocsConfig;
-      if (!parsed.url) {
-        throw new Error(`Config file ${filepath} is missing required "url" field`);
+  while (true) {
+    for (const filename of CONFIG_FILENAMES) {
+      const filepath = resolve(searchDir, filename);
+      try {
+        const content = await readFile(filepath, 'utf-8');
+        const parsed = parseYaml(content) as AgentDocsConfig;
+        if (!parsed.url) {
+          throw new Error(`Config file ${filepath} is missing required "url" field`);
+        }
+        return parsed;
+      } catch (err) {
+        if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue;
+        throw err;
       }
-      return parsed;
-    } catch (err) {
-      if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue;
-      throw err;
     }
+
+    const parent = dirname(searchDir);
+    if (parent === searchDir) break; // reached filesystem root
+    searchDir = parent;
   }
 
   throw new Error(

src/helpers/vitest-runner.ts

@@ -1,17 +1,34 @@
-import { describe, it, expect } from 'vitest';
+import { describe, it, expect, beforeAll } from 'vitest';
 import { runChecks } from '../runner.js';
 import { loadConfig } from './config.js';
-import type { AgentDocsConfig, CheckResult } from '../types.js';
+import { getChecksSorted } from '../checks/registry.js';
+import type { AgentDocsConfig, CheckResult, ReportResult } from '../types.js';
 
 // Ensure all checks are registered
 import '../checks/index.js';
 
+const STATUS_ICON: Record<string, string> = {
+  pass: '\u2713',
+  warn: '\u26A0',
+  fail: '\u2717',
+  skip: '\u2192',
+  error: '!',
+};
+
+function resolveConfig(
+  configOrDir: AgentDocsConfig | string | undefined,
+): Promise<AgentDocsConfig> {
+  if (typeof configOrDir === 'object') return Promise.resolve(configOrDir);
+  return loadConfig(typeof configOrDir === 'string' ? configOrDir : undefined);
+}
+
 /**
  * Auto-generates vitest tests from an agent-docs config file.
+ * Produces a single pass/fail test with a summary of all check results.
  *
- * Usage in a test file:
+ * Usage:
  * ```ts
- * import { describeAgentDocs } from 'agent-docs-testing/helpers';
+ * import { describeAgentDocs } from 'afdocs/helpers';
  * describeAgentDocs();
  * ```
  */
@@ -20,17 +37,14 @@ export function describeAgentDocs(configOrDir?: AgentDocsConfig | string): void
     let results: CheckResult[];
 
     it('should run checks', async () => {
-      const config =
-        typeof configOrDir === 'object'
-          ? configOrDir
-          : await loadConfig(typeof configOrDir === 'string' ? configOrDir : undefined);
+      const config = await resolveConfig(configOrDir);
 
       const report = await runChecks(config.url, {
         checkIds: config.checks,
         ...config.options,
       });
       results = report.results;
-    });
+    }, 120_000);
 
     it('should have no failing checks', () => {
       expect(results).toBeDefined();
@@ -45,34 +59,54 @@ export function describeAgentDocs(configOrDir?: AgentDocsConfig | string): void
 
 /**
  * Auto-generates individual vitest tests per check from an agent-docs config.
+ * Each check appears as its own test line in CI output, giving clear visibility
+ * into which checks passed, warned, failed, or were skipped.
  *
  * Usage:
  * ```ts
- * import { describeAgentDocsPerCheck } from 'agent-docs-testing/helpers';
+ * import { describeAgentDocsPerCheck } from 'afdocs/helpers';
  * describeAgentDocsPerCheck();
  * ```
  */
 export function describeAgentDocsPerCheck(configOrDir?: AgentDocsConfig | string): void {
   describe('Agent-Friendly Documentation', () => {
-    let _results: Map<string, CheckResult>;
-
-    // Run all checks once, then assert per-check
-    it('should run checks', async () => {
-      const config =
-        typeof configOrDir === 'object'
-          ? configOrDir
-          : await loadConfig(typeof configOrDir === 'string' ? configOrDir : undefined);
+    let report: ReportResult;
+    let resultsByCheck: Map<string, CheckResult>;
 
-      const report = await runChecks(config.url, {
+    // Run all checks once upfront
+    beforeAll(async () => {
+      const config = await resolveConfig(configOrDir);
+      report = await runChecks(config.url, {
         checkIds: config.checks,
         ...config.options,
       });
-      _results = new Map(report.results.map((r) => [r.id, r]));
-    });
+      resultsByCheck = new Map(report.results.map((r) => [r.id, r]));
+    }, 120_000);
+
+    // Register a test per known check. Checks not included in the run
+    // (filtered via config.checks) are reported as skipped rather than
+    // silently passing, so CI output accurately reflects what was tested.
+    const allChecks = getChecksSorted();
 
-    // Individual check assertions will be generated after the setup test runs.
-    // Since vitest doesn't support dynamic test generation after suite setup,
-    // users should use describeAgentDocs() for the simple case or import
-    // checks individually for per-check control.
+    for (const check of allChecks) {
+      it(check.id, (ctx) => {
+        const result = resultsByCheck?.get(check.id);
+        if (!result) {
+          // Check was filtered out by config
+          ctx.skip();
+          return;
+        }
+
+        const icon = STATUS_ICON[result.status] ?? '?';
+        console.log(`${icon} [${result.status}] ${result.message}`);
+
+        if (result.status === 'fail') {
+          expect.fail(`${result.message}`);
+        }
+        if (result.status === 'error') {
+          expect.fail(`Check error: ${result.message}`);
+        }
+      });
+    }
   });
 }

test/unit/helpers/config.test.ts

@@ -51,4 +51,31 @@ describe('loadConfig', () => {
 
     await expect(loadConfig(TMP_DIR)).rejects.toThrow('missing required "url" field');
   });
+
+  it('walks up directories to find config', async () => {
+    const parentDir = TMP_DIR;
+    const childDir = resolve(TMP_DIR, 'sub/nested');
+    await mkdir(childDir, { recursive: true });
+    await writeFile(
+      resolve(parentDir, 'agent-docs.config.yml'),
+      'url: https://parent.example.com\n',
+    );
+
+    const config = await loadConfig(childDir);
+    expect(config.url).toBe('https://parent.example.com');
+  });
+
+  it('finds config in immediate dir before walking up', async () => {
+    const parentDir = TMP_DIR;
+    const childDir = resolve(TMP_DIR, 'sub');
+    await mkdir(childDir, { recursive: true });
+    await writeFile(
+      resolve(parentDir, 'agent-docs.config.yml'),
+      'url: https://parent.example.com\n',
+    );
+    await writeFile(resolve(childDir, 'agent-docs.config.yml'), 'url: https://child.example.com\n');
+
+    const config = await loadConfig(childDir);
+    expect(config.url).toBe('https://child.example.com');
+  });
 });