Merge pull request #36 from agent-ecosystem/sitemap-discovery-and-filtering

Sitemap discovery and filtering fixups

Author: dacharyc

docs/reference/cli.md

@@ -107,6 +107,34 @@ afdocs check https://docs.example.com --max-links 10
 afdocs check https://docs.example.com --max-links 100
 ```
 
+### URL discovery
+
+| Flag                      | Default     | Description                                                      |
+| ------------------------- | ----------- | ---------------------------------------------------------------- |
+| `--doc-locale <code>`     | auto-detect | Preferred locale for URL discovery (e.g. `en`, `fr`, `ja`)       |
+| `--doc-version <version>` | auto-detect | Preferred version for URL discovery (e.g. `v3`, `2.x`, `latest`) |
+
+When `afdocs` discovers pages from a sitemap or `llms.txt`, it automatically filters out duplicate locale and version variants so you get a representative sample of unique content.
+
+The resolution order for both flags is:
+
+1. **Explicit flag** (`--doc-locale`, `--doc-version`) if provided
+2. **Auto-detect** from the base URL path (e.g. `https://docs.example.com/fr/v3` detects `fr` and `v3`)
+3. **Built-in fallback** when neither of the above yields a value: locale falls back to `en`; version prefers unversioned URLs, then `latest`/`stable`/`current`, then the highest semver. Pre-release channels (`dev`, `next`, `nightly`, `canary`) are ranked below stable versions
+
+Use the flags when the base URL doesn't contain locale or version segments but the site organizes content by locale or version.
+
+```bash
+# Prefer French locale during discovery
+afdocs check https://docs.example.com --doc-locale fr
+
+# Prefer a specific version
+afdocs check https://docs.example.com --doc-version v3
+
+# Both together
+afdocs check https://docs.example.com --doc-locale ja --doc-version 2.x
+```
+
 ### Request behavior
 
 | Flag                    | Default | Description                            |

docs/reference/config-file.md

@@ -25,6 +25,8 @@ options:
   samplingStrategy: deterministic
   maxConcurrency: 5
   requestDelay: 100
+  preferredLocale: en
+  preferredVersion: v3
   thresholds:
     pass: 50000
     fail: 100000
@@ -52,15 +54,17 @@ This is particularly useful when your docs platform doesn't support certain capa
 
 Override default runner options. All fields are optional:
 
-| Field              | Default  | Description                                          |
-| ------------------ | -------- | ---------------------------------------------------- |
-| `maxLinksToTest`   | `50`     | Maximum number of pages to sample                    |
-| `samplingStrategy` | `random` | `random`, `deterministic`, `curated`, or `none`      |
-| `maxConcurrency`   | `3`      | Maximum concurrent HTTP requests                     |
-| `requestDelay`     | `200`    | Delay between requests in milliseconds               |
-| `requestTimeout`   | `30000`  | Timeout for individual HTTP requests in milliseconds |
-| `thresholds.pass`  | `50000`  | Page size pass threshold in characters               |
-| `thresholds.fail`  | `100000` | Page size fail threshold in characters               |
+| Field              | Default     | Description                                                |
+| ------------------ | ----------- | ---------------------------------------------------------- |
+| `maxLinksToTest`   | `50`        | Maximum number of pages to sample                          |
+| `samplingStrategy` | `random`    | `random`, `deterministic`, `curated`, or `none`            |
+| `maxConcurrency`   | `3`         | Maximum concurrent HTTP requests                           |
+| `requestDelay`     | `200`       | Delay between requests in milliseconds                     |
+| `requestTimeout`   | `30000`     | Timeout for individual HTTP requests in milliseconds       |
+| `preferredLocale`  | auto-detect | Preferred locale for URL discovery (e.g. `en`, `fr`, `ja`) |
+| `preferredVersion` | auto-detect | Preferred version for URL discovery (e.g. `v3`, `2.x`)     |
+| `thresholds.pass`  | `50000`     | Page size pass threshold in characters                     |
+| `thresholds.fail`  | `100000`    | Page size fail threshold in characters                     |
 
 ### `pages` (optional)
 

src/checks/observability/llms-txt-freshness.ts

@@ -256,12 +256,11 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
   //    sitemap URLs at the redirected host are accepted rather than filtered out.
   const effectiveOrigin = ctx.effectiveOrigin ?? ctx.origin;
   const sitemapWarnings: string[] = [];
-  let sitemapUrls = await getUrlsFromSitemap(
-    ctx,
-    sitemapWarnings,
-    MAX_FRESHNESS_SITEMAP_URLS,
-    effectiveOrigin,
-  );
+  let sitemapUrls = await getUrlsFromSitemap(ctx, sitemapWarnings, {
+    maxUrls: MAX_FRESHNESS_SITEMAP_URLS,
+    originOverride: effectiveOrigin,
+    skipRefinement: true,
+  });
   let sitemapSource = 'robots.txt/sitemap.xml';
   const baseUrlPath = new URL(ctx.baseUrl).pathname.replace(/\/$/, '');
 

src/cli/commands/check.ts

@@ -30,6 +30,8 @@ export function registerCheckCommand(program: Command): void {
       '--urls <urls>',
       'Comma-separated page URLs for curated scoring (implies --sampling curated)',
     )
+    .option('--doc-locale <code>', 'Preferred locale for URL discovery (e.g. en, fr, ja)')
+    .option('--doc-version <version>', 'Preferred version for URL discovery (e.g. v3, 2.x, latest)')
     .option('--pass-threshold <n>', 'Pass threshold in characters')
     .option('--fail-threshold <n>', 'Fail threshold in characters')
     .option('-v, --verbose', 'Show per-page details for checks with issues')
@@ -163,6 +165,11 @@ export function registerCheckCommand(program: Command): void {
         process.stderr.write(`Running checks on ${target}...\n`);
       }
 
+      const preferredLocale =
+        (opts.docLocale as string | undefined) ?? config?.options?.preferredLocale;
+      const preferredVersion =
+        (opts.docVersion as string | undefined) ?? config?.options?.preferredVersion;
+
       const report = await runChecks(url, {
         checkIds,
         maxConcurrency,
@@ -174,6 +181,8 @@ export function registerCheckCommand(program: Command): void {
           pass: passThreshold,
           fail: failThreshold,
         },
+        ...(preferredLocale && { preferredLocale }),
+        ...(preferredVersion && { preferredVersion }),
       });
 
       let output: string;