Treat www and bare-host as same origin in sitemap filtering

Sitemap entries are commonly published on the bare-host canonical
(e.g. https://swift.org/...) even when the served site is www.swift.org.
The strict `origin !==` comparison in shouldInclude() and scopeUrls()
discarded every such URL, causing afdocs to fall back to single-page
sampling and trigger the single-page-sample diagnostic.

PR #82 already added the right sitemap discovery candidates, so the
root sitemap was being fetched — its URLs were just being filtered
out before they could be used.

Fix: introduce isSameOriginIgnoringWww() (built on the existing
isWwwVariant helper) and use it in both filter sites. Adds tests
covering both directions of www mismatch and a regression test
confirming truly cross-host URLs are still rejected.

Fixes #83.

Author: dacharyc

src/checks/observability/llms-txt-coverage.ts

@@ -2,6 +2,7 @@ import { registerCheck } from '../registry.js';
 import {
   getUrlsFromCachedLlmsTxtWithOmitted,
   getUrlsFromSitemap,
+  isSameOriginIgnoringWww,
   parseSitemapUrls,
 } from '../../helpers/get-page-urls.js';
 import { isNonPageUrl } from '../../helpers/to-md-urls.js';
@@ -297,7 +298,7 @@ function scopeUrls(urls: string[], origin: string, baseUrlPath: string): string[
   return urls.filter((url) => {
     try {
       const parsed = new URL(url);
-      if (parsed.origin !== origin) return false;
+      if (!isSameOriginIgnoringWww(parsed.origin, origin)) return false;
       if (baseUrlPath && baseUrlPath !== '/') {
         if (!parsed.pathname.startsWith(baseUrlPath + '/') && parsed.pathname !== baseUrlPath) {
           return false;

src/helpers/get-page-urls.ts

@@ -740,7 +740,7 @@ export async function getUrlsFromSitemap(
   function shouldInclude(url: string): boolean {
     try {
       const u = new URL(url);
-      if (u.origin !== matchOrigin) return false;
+      if (!isSameOriginIgnoringWww(u.origin, matchOrigin)) return false;
       if (prefixPath) return matchesPathPrefix(url, prefixPath);
       return true;
     } catch {
@@ -804,6 +804,25 @@ function isWwwVariant(hostname1: string, hostname2: string): boolean {
   return hostname1 === `www.${hostname2}` || hostname2 === `www.${hostname1}`;
 }
 
+/**
+ * Compare two origins, treating `www.host` and `host` as equivalent.
+ *
+ * Sitemap entries are commonly published on the bare-host canonical
+ * (e.g. `https://swift.org/...`) even when the served site is `www.swift.org`.
+ * Strict origin equality would discard every such URL.
+ */
+export function isSameOriginIgnoringWww(origin1: string, origin2: string): boolean {
+  if (origin1 === origin2) return true;
+  try {
+    const a = new URL(origin1);
+    const b = new URL(origin2);
+    if (a.protocol !== b.protocol || a.port !== b.port) return false;
+    return isWwwVariant(a.hostname, b.hostname);
+  } catch {
+    return false;
+  }
+}
+
 /**
  * Get the base URL for path-prefix filtering, accounting for cross-host redirects.
  *

test/unit/checks/llms-txt-coverage.test.ts

@@ -708,6 +708,38 @@ describe('llms-txt-coverage', () => {
     expect(result.details?.sitemapDocPages).toBe(2);
   });
 
+  test('scopes sitemap URLs across www vs bare-host (issue #83)', async () => {
+    // swift.org-style: scored URL is `www.host`, but sitemap entries are on bare host.
+    // Coverage scoping must treat these as same-origin or coverage = 0%.
+    const wwwHost = 'www.www-cov.local';
+    const bareHost = 'www-cov.local';
+    const llmsTxtPages = [`http://${wwwHost}/docs/intro`, `http://${wwwHost}/docs/guide`];
+    const sitemapPages = [
+      `http://${bareHost}/docs/intro`,
+      `http://${bareHost}/docs/guide`,
+      `http://${bareHost}/docs/extra`,
+    ];
+
+    const ctx = makeCtx(wwwHost, llmsTxtPages, '/docs');
+
+    server.use(
+      http.get(
+        `http://${wwwHost}/robots.txt`,
+        () => new HttpResponse(`Sitemap: http://${wwwHost}/sitemap.xml`, { status: 200 }),
+      ),
+      http.get(
+        `http://${wwwHost}/sitemap.xml`,
+        () =>
+          new HttpResponse(makeSitemap(sitemapPages), {
+            headers: { 'content-type': 'application/xml' },
+          }),
+      ),
+    );
+
+    const result = await check.run(ctx);
+    expect(result.details?.sitemapDocPages).toBe(3);
+  });
+
   test('excludes paths relative to base URL prefix', async () => {
     const host = 'basepath-exclude.local';
     const pages = [`http://${host}/docs/getting-started`, `http://${host}/docs/api-reference`];

test/unit/helpers/get-page-urls.test.ts

@@ -14,6 +14,7 @@ import {
   deduplicateVersionedUrls,
   extractVersionFromUrl,
   extractLocaleFromUrl,
+  isSameOriginIgnoringWww,
 } from '../../../src/helpers/get-page-urls.js';
 import { MAX_SITEMAP_URLS } from '../../../src/constants.js';
 import { createContext } from '../../../src/runner.js';
@@ -156,6 +157,33 @@ describe('filterByPathPrefix', () => {
   });
 });
 
+describe('isSameOriginIgnoringWww', () => {
+  it('returns true for identical origins', () => {
+    expect(isSameOriginIgnoringWww('https://example.com', 'https://example.com')).toBe(true);
+  });
+
+  it('returns true for www vs bare-host (issue #83)', () => {
+    expect(isSameOriginIgnoringWww('https://swift.org', 'https://www.swift.org')).toBe(true);
+    expect(isSameOriginIgnoringWww('https://www.swift.org', 'https://swift.org')).toBe(true);
+  });
+
+  it('returns false for different protocols', () => {
+    expect(isSameOriginIgnoringWww('http://example.com', 'https://example.com')).toBe(false);
+  });
+
+  it('returns false for different ports', () => {
+    expect(isSameOriginIgnoringWww('https://example.com:8443', 'https://example.com')).toBe(false);
+  });
+
+  it('returns false for unrelated hosts', () => {
+    expect(isSameOriginIgnoringWww('https://example.com', 'https://other.com')).toBe(false);
+  });
+
+  it('returns false for subdomains that are not www (e.g. docs)', () => {
+    expect(isSameOriginIgnoringWww('https://docs.example.com', 'https://example.com')).toBe(false);
+  });
+});
+
 describe('getPathFilterBase', () => {
   it('returns baseUrl when no effectiveOrigin is set', () => {
     const ctx = createContext('https://example.com/docs', { requestDelay: 0 });
@@ -1207,6 +1235,99 @@ describe('getPageUrls', () => {
     ]);
   });
 
+  it('accepts sitemap URLs published on bare-host when scored URL has www (issue #83)', async () => {
+    // swift.org-style: scored URL is www.host.local, but the sitemap lists URLs
+    // on the bare host. Without www-equivalence in the origin filter, every URL
+    // is discarded and afdocs falls back to single-page sampling.
+    mockSitemapNotFound(server, 'http://www.www-bare.local/documentation/');
+    server.use(
+      http.get(
+        'http://www.www-bare.local/robots.txt',
+        () => new HttpResponse('User-agent: *\n', { status: 200 }),
+      ),
+      http.get(
+        'http://www.www-bare.local/sitemap.xml',
+        () =>
+          new HttpResponse(
+            `<?xml version="1.0"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>http://www-bare.local/documentation/intro</loc></url>
+  <url><loc>http://www-bare.local/documentation/guide</loc></url>
+</urlset>`,
+            { status: 200, headers: { 'Content-Type': 'application/xml' } },
+          ),
+      ),
+    );
+
+    const ctx = createContext('http://www.www-bare.local/documentation/', { requestDelay: 0 });
+    const warnings: string[] = [];
+    const result = await getUrlsFromSitemap(ctx, warnings, { skipRefinement: true });
+    expect(result).toEqual([
+      'http://www-bare.local/documentation/intro',
+      'http://www-bare.local/documentation/guide',
+    ]);
+  });
+
+  it('accepts sitemap URLs published on www-host when scored URL is bare (issue #83)', async () => {
+    // Inverse scenario: scored URL is bare host, sitemap entries are www-prefixed.
+    mockSitemapNotFound(server, 'http://bare-www.local');
+    server.use(
+      http.get(
+        'http://bare-www.local/robots.txt',
+        () => new HttpResponse('User-agent: *\n', { status: 200 }),
+      ),
+      http.get(
+        'http://bare-www.local/sitemap.xml',
+        () =>
+          new HttpResponse(
+            `<?xml version="1.0"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>http://www.bare-www.local/page-1</loc></url>
+  <url><loc>http://www.bare-www.local/page-2</loc></url>
+</urlset>`,
+            { status: 200, headers: { 'Content-Type': 'application/xml' } },
+          ),
+      ),
+    );
+
+    const ctx = createContext('http://bare-www.local', { requestDelay: 0 });
+    const warnings: string[] = [];
+    const result = await getUrlsFromSitemap(ctx, warnings, { skipRefinement: true });
+    expect(result).toEqual([
+      'http://www.bare-www.local/page-1',
+      'http://www.bare-www.local/page-2',
+    ]);
+  });
+
+  it('still rejects truly cross-host sitemap URLs', async () => {
+    // Sanity check: www-equivalence does not relax filtering for unrelated hosts.
+    mockSitemapNotFound(server, 'http://strict-host.local');
+    server.use(
+      http.get(
+        'http://strict-host.local/robots.txt',
+        () => new HttpResponse('User-agent: *\n', { status: 200 }),
+      ),
+      http.get(
+        'http://strict-host.local/sitemap.xml',
+        () =>
+          new HttpResponse(
+            `<?xml version="1.0"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>http://strict-host.local/keep</loc></url>
+  <url><loc>http://other-host.local/drop</loc></url>
+  <url><loc>https://strict-host.local/drop-scheme</loc></url>
+</urlset>`,
+            { status: 200, headers: { 'Content-Type': 'application/xml' } },
+          ),
+      ),
+    );
+
+    const ctx = createContext('http://strict-host.local', { requestDelay: 0 });
+    const warnings: string[] = [];
+    const result = await getUrlsFromSitemap(ctx, warnings, { skipRefinement: true });
+    expect(result).toEqual(['http://strict-host.local/keep']);
+  });
+
   it('warns and skips gzipped sitemap from robots.txt', async () => {
     server.use(
       http.get(