Merge pull request #41 from philip/canonical-origin-rewrite

feat: add --canonical-origin flag for preview/staging testing

Author: dacharyc

docs/reference/cli.md

@@ -137,10 +137,11 @@ afdocs check https://docs.example.com --doc-locale ja --doc-version 2.x
 
 ### Request behavior
 
-| Flag                    | Default | Description                            |
-| ----------------------- | ------- | -------------------------------------- |
-| `--max-concurrency <n>` | `3`     | Maximum concurrent HTTP requests       |
-| `--request-delay <ms>`  | `200`   | Delay between requests in milliseconds |
+| Flag                       | Default | Description                                 |
+| -------------------------- | ------- | ------------------------------------------- |
+| `--max-concurrency <n>`    | `3`     | Maximum concurrent HTTP requests            |
+| `--request-delay <ms>`     | `200`   | Delay between requests in milliseconds      |
+| `--canonical-origin <url>` |         | The production domain your content links to |
 
 AFDocs enforces delays between requests and caps concurrent connections to avoid overloading your server. Adjust these if you need gentler or faster runs:
 
@@ -152,6 +153,13 @@ afdocs check https://docs.example.com --request-delay 500 --max-concurrency 1
 afdocs check https://docs.example.com --request-delay 50 --max-concurrency 10
 ```
 
+Use `--canonical-origin` when your site's URLs in `sitemap.xml` and `llms.txt` don't match the domain you're testing, such as preview deployments or localhost.
+
+```bash
+# Test a preview deployment
+afdocs check https://preview-xyz-example.app/docs --canonical-origin https://example.com
+```
+
 ### Size thresholds
 
 | Flag                   | Default  | Description                            |

docs/reference/config-file.md

@@ -27,6 +27,7 @@ options:
   requestDelay: 100
   preferredLocale: en
   preferredVersion: v3
+  canonicalOrigin: https://example.com
   thresholds:
     pass: 50000
     fail: 100000
@@ -63,6 +64,7 @@ Override default runner options. All fields are optional:
 | `requestTimeout`   | `30000`     | Timeout for individual HTTP requests in milliseconds       |
 | `preferredLocale`  | auto-detect | Preferred locale for URL discovery (e.g. `en`, `fr`, `ja`) |
 | `preferredVersion` | auto-detect | Preferred version for URL discovery (e.g. `v3`, `2.x`)     |
+| `canonicalOrigin`  |             | The production domain your content links to                |
 | `thresholds.pass`  | `50000`     | Page size pass threshold in characters                     |
 | `thresholds.fail`  | `100000`    | Page size fail threshold in characters                     |
 

src/cli/commands/check.ts

@@ -37,6 +37,10 @@ export function registerCheckCommand(program: Command): void {
     .option('-v, --verbose', 'Show per-page details for checks with issues')
     .option('--fixes', 'Show fix suggestions for warn/fail checks')
     .option('--score', 'Include scoring data in JSON output')
+    .option(
+      '--canonical-origin <url>',
+      'Rewrite this origin to the target in fetched content (for preview/staging testing)',
+    )
     .action(async (rawUrl: string | undefined, opts: Record<string, unknown>) => {
       // Load config: explicit path or auto-discover
       let config;
@@ -170,6 +174,19 @@ export function registerCheckCommand(program: Command): void {
       const preferredVersion =
         (opts.docVersion as string | undefined) ?? config?.options?.preferredVersion;
 
+      let canonicalOrigin: string | undefined;
+      const rawCanonical =
+        (opts.canonicalOrigin as string | undefined) ?? config?.options?.canonicalOrigin;
+      if (rawCanonical) {
+        try {
+          canonicalOrigin = new URL(normalizeUrl(rawCanonical)).origin;
+        } catch {
+          process.stderr.write(`Error: Invalid --canonical-origin URL "${rawCanonical}".\n`);
+          process.exitCode = 1;
+          return;
+        }
+      }
+
       const report = await runChecks(url, {
         checkIds,
         maxConcurrency,
@@ -183,6 +200,7 @@ export function registerCheckCommand(program: Command): void {
         },
         ...(preferredLocale && { preferredLocale }),
         ...(preferredVersion && { preferredVersion }),
+        ...(canonicalOrigin && { canonicalOrigin }),
       });
 
       let output: string;

src/http.ts

@@ -11,13 +11,23 @@ interface RateLimitedHttpClientOptions {
   requestDelay: number;
   requestTimeout: number;
   maxConcurrency: number;
+  canonicalOrigin?: string;
+  targetOrigin?: string;
 }
 
 const MAX_RETRIES = 2;
 
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
 export function createHttpClient(options: RateLimitedHttpClientOptions): HttpClient {
   let lastRequestTime = 0;
   let activeRequests = 0;
+  const originPattern =
+    options.canonicalOrigin && options.targetOrigin
+      ? new RegExp(escapeRegExp(options.canonicalOrigin), 'g')
+      : null;
 
   async function waitForSlot(): Promise<void> {
     // Wait for concurrency slot
@@ -66,6 +76,24 @@ export function createHttpClient(options: RateLimitedHttpClientOptions): HttpCli
             }
           }
 
+          if (originPattern && options.targetOrigin) {
+            const ct = response.headers.get('content-type') ?? '';
+            if (/text|xml|json|markdown/.test(ct)) {
+              const body = await response.text();
+              originPattern.lastIndex = 0;
+              const rewritten = body.replace(originPattern, options.targetOrigin);
+              return {
+                ok: response.ok,
+                status: response.status,
+                statusText: response.statusText,
+                headers: response.headers,
+                url: response.url,
+                redirected: response.redirected,
+                text: async () => rewritten,
+              } as HttpResponse;
+            }
+          }
+
           return response as HttpResponse;
         } finally {
           activeRequests--;

src/runner.ts

@@ -53,6 +53,8 @@ export function createContext(baseUrl: string, options?: Partial<RunnerOptions>)
       requestDelay: merged.requestDelay,
       requestTimeout: merged.requestTimeout,
       maxConcurrency: merged.maxConcurrency,
+      canonicalOrigin: merged.canonicalOrigin,
+      targetOrigin: merged.canonicalOrigin ? url.origin : undefined,
     }),
     options: merged,
     pageCache: new Map(),

src/types.ts

@@ -82,6 +82,8 @@ export interface CheckOptions {
   preferredLocale?: string;
   /** Preferred version for URL discovery (e.g. 'v3', '2.x', 'latest'). Overrides auto-detection from baseUrl. */
   preferredVersion?: string;
+  /** Canonical origin to rewrite in fetched content (for preview/staging testing). */
+  canonicalOrigin?: string;
 }
 
 export interface SizeThresholds {

test/unit/helpers/http.test.ts

@@ -89,4 +89,87 @@ describe('createHttpClient', () => {
     expect(response.status).toBe(429);
     expect(callCount).toBe(1);
   });
+
+  describe('origin rewriting', () => {
+    function makeTextResponse(
+      body: string,
+      opts: { status?: number; contentType?: string; url?: string; redirected?: boolean } = {},
+    ): Response {
+      const status = opts.status ?? 200;
+      return {
+        ok: status >= 200 && status < 300,
+        status,
+        statusText: 'OK',
+        headers: new Headers(opts.contentType ? { 'content-type': opts.contentType } : undefined),
+        url: opts.url ?? 'http://preview.local/docs/llms.txt',
+        redirected: opts.redirected ?? false,
+        text: async () => body,
+      } as unknown as Response;
+    }
+
+    it('rewrites all occurrences of canonicalOrigin in text responses', async () => {
+      const body = [
+        '- [Guide](https://prod.example.com/docs/guide)',
+        '- [API](https://prod.example.com/docs/api)',
+        'All pages: https://prod.example.com/docs/llms.txt',
+      ].join('\n');
+      globalThis.fetch = vi.fn(async () => makeTextResponse(body, { contentType: 'text/plain' }));
+
+      const client = createHttpClient({
+        requestDelay: 0,
+        requestTimeout: 5000,
+        maxConcurrency: 10,
+        canonicalOrigin: 'https://prod.example.com',
+        targetOrigin: 'https://preview.local',
+      });
+      const response = await client.fetch('http://preview.local/docs/llms.txt');
+      const text = await response.text();
+
+      expect(text).not.toContain('prod.example.com');
+      expect(text).toContain('https://preview.local/docs/guide');
+      expect(text).toContain('https://preview.local/docs/api');
+      expect(text).toContain('https://preview.local/docs/llms.txt');
+    });
+
+    it('preserves url and redirected from original response', async () => {
+      globalThis.fetch = vi.fn(async () =>
+        makeTextResponse('https://prod.example.com/page', {
+          contentType: 'text/plain',
+          url: 'http://preview.local/docs/llms.txt',
+          redirected: true,
+        }),
+      );
+
+      const client = createHttpClient({
+        requestDelay: 0,
+        requestTimeout: 5000,
+        maxConcurrency: 10,
+        canonicalOrigin: 'https://prod.example.com',
+        targetOrigin: 'https://preview.local',
+      });
+      const response = await client.fetch('http://preview.local/docs/llms.txt');
+
+      expect(response.url).toBe('http://preview.local/docs/llms.txt');
+      expect(response.redirected).toBe(true);
+    });
+
+    it('skips rewrite for non-text content types', async () => {
+      const original = 'https://prod.example.com/binary-data';
+      globalThis.fetch = vi.fn(async () =>
+        makeTextResponse(original, { contentType: 'application/octet-stream' }),
+      );
+
+      const client = createHttpClient({
+        requestDelay: 0,
+        requestTimeout: 5000,
+        maxConcurrency: 10,
+        canonicalOrigin: 'https://prod.example.com',
+        targetOrigin: 'https://preview.local',
+      });
+      const response = await client.fetch('http://preview.local/file.bin');
+      const text = await response.text();
+
+      expect(text).toBe(original);
+    });
+  });
 });