Fix false positives, improve verbose output, and cap

   HTTP retries

   - Skip non-page file types (.json, .xml) in content-negotiation check
   - Add alreadyMd flag to markdown-url-support so .md URLs that serve
     HTML get a distinct message instead of 'no .md URL found'
   - Improve content-start-position heuristic: add linkDensity and
     headingFollowedByContent look-ahead to skip nav/sidebar headings,
     tighten prose fallback thresholds, fix CSS pattern to match custom
     properties with digits (fixes Stripe 0% false positive)
   - Ignore code fences inside markdown table cells (vendor extension)
   - Make all verbose formatter messages descriptive: content-negotiation
     shows actual content-type, markdown-url-support distinguishes .md
     URLs serving HTML, add per-page detail for markdown-code-fence-validity
   - Cap HTTP 429 retries at 2 to prevent infinite retry loops that caused
     the CLI to hang on rate-limited sites like Stripe
   - Add tests for all new behavior

Author: dacharyc

src/checks/content-structure/markdown-code-fence-validity.ts

@@ -40,6 +40,10 @@ function analyzeFences(content: string): { fenceCount: number; issues: FenceIssu
     const match = FENCE_RE.exec(stripped);
     if (!match) continue;
 
+    // Skip fences inside markdown table cells (e.g. "``` | ```" or "| ```")
+    // These aren't real CommonMark fences — multi-line table cells are a vendor extension
+    if (stripped.includes('|')) continue;
+
     const char = match[3] ? '`' : '~';
     const length = (match[3] || match[4]).length;
 

src/checks/markdown-availability/content-negotiation.ts

@@ -1,13 +1,15 @@
 import { registerCheck } from '../registry.js';
 import { looksLikeMarkdown } from '../../helpers/detect-markdown.js';
 import { discoverAndSamplePages } from '../../helpers/get-page-urls.js';
+import { isNonPageUrl } from '../../helpers/to-md-urls.js';
 import type { CheckContext, CheckResult } from '../../types.js';
 
 type Classification = 'markdown-with-correct-type' | 'markdown-with-wrong-type' | 'html';
 
 interface PageResult {
   url: string;
   classification: Classification;
+  skipped?: boolean;
   contentType: string;
   status: number;
   error?: string;
@@ -31,6 +33,10 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
     const batch = pageUrls.slice(i, i + concurrency);
     const batchResults = await Promise.all(
       batch.map(async (url): Promise<PageResult> => {
+        // Non-page file types (e.g. .json, .xml) are already in a machine-readable format
+        if (isNonPageUrl(url)) {
+          return { url, classification: 'html', skipped: true, contentType: '', status: 0 };
+        }
         try {
           const response = await ctx.http.fetch(url, {
             headers: { Accept: 'text/markdown' },
@@ -77,16 +83,21 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
     results.push(...batchResults);
   }
 
-  const markdownWithCorrectType = results.filter(
+  const testedResults = results.filter((r) => !r.skipped);
+  const skippedCount = results.length - testedResults.length;
+  const markdownWithCorrectType = testedResults.filter(
     (r) => r.classification === 'markdown-with-correct-type',
   ).length;
-  const markdownWithWrongType = results.filter(
+  const markdownWithWrongType = testedResults.filter(
     (r) => r.classification === 'markdown-with-wrong-type',
   ).length;
-  const htmlOnly = results.filter((r) => r.classification === 'html').length;
-  const negotiationRate = Math.round((markdownWithCorrectType / results.length) * 100);
-  const fetchErrors = results.filter((r) => r.error).length;
-  const rateLimited = results.filter((r) => r.status === 429).length;
+  const htmlOnly = testedResults.filter((r) => r.classification === 'html').length;
+  const negotiationRate =
+    testedResults.length > 0
+      ? Math.round((markdownWithCorrectType / testedResults.length) * 100)
+      : 0;
+  const fetchErrors = testedResults.filter((r) => r.error).length;
+  const rateLimited = testedResults.filter((r) => r.status === 429).length;
 
   const pageLabel = wasSampled ? 'sampled pages' : 'pages';
   const suffix =
@@ -95,7 +106,8 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
 
   const details: Record<string, unknown> = {
     totalPages,
-    testedPages: results.length,
+    testedPages: testedResults.length,
+    skippedPages: skippedCount,
     sampled: wasSampled,
     markdownWithCorrectType,
     markdownWithWrongType,
@@ -112,7 +124,7 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
       id,
       category,
       status: 'pass',
-      message: `${markdownWithCorrectType}/${results.length} ${pageLabel} support content negotiation (${negotiationRate}%)${suffix}`,
+      message: `${markdownWithCorrectType}/${testedResults.length} ${pageLabel} support content negotiation (${negotiationRate}%)${suffix}`,
       details,
     };
   }
@@ -131,7 +143,7 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
     id,
     category,
     status: 'fail',
-    message: `Server ignores Accept: text/markdown header (0/${results.length} ${pageLabel} return markdown)${suffix}`,
+    message: `Server ignores Accept: text/markdown header (0/${testedResults.length} ${pageLabel} return markdown)${suffix}`,
     details,
   };
 }

src/checks/markdown-availability/markdown-url-support.ts

@@ -8,6 +8,8 @@ interface PageResult {
   url: string;
   mdUrl: string;
   supported: boolean;
+  skipped?: boolean;
+  alreadyMd?: boolean;
   status: number;
   error?: string;
 }
@@ -31,6 +33,11 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
     const batchResults = await Promise.all(
       batch.map(async (url): Promise<PageResult> => {
         const candidates = toMdUrls(url);
+        // Non-markdown file types (e.g. .json, .xml) have no .md equivalent — skip them
+        if (candidates.length === 0) {
+          return { url, mdUrl: url, supported: false, skipped: true, status: 0 };
+        }
+        const alreadyMd = /\.mdx?$/i.test(new URL(url).pathname);
         let lastError: string | undefined;
         for (const mdUrl of candidates) {
           try {
@@ -46,24 +53,34 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
                 url,
                 markdown: { content: body, source: 'md-url' },
               });
-              return { url, mdUrl, supported: true, status: response.status };
+              return { url, mdUrl, supported: true, alreadyMd, status: response.status };
             }
             lastError = undefined; // Got a response, not a fetch error
           } catch (err) {
             lastError = err instanceof Error ? err.message : String(err);
           }
         }
-        return { url, mdUrl: candidates[0], supported: false, status: 0, error: lastError };
+        return {
+          url,
+          mdUrl: candidates[0],
+          supported: false,
+          alreadyMd,
+          status: 0,
+          error: lastError,
+        };
       }),
     );
     results.push(...batchResults);
   }
 
-  const mdSupported = results.filter((r) => r.supported).length;
-  const mdUnsupported = results.length - mdSupported;
-  const supportRate = Math.round((mdSupported / results.length) * 100);
-  const fetchErrors = results.filter((r) => r.error).length;
-  const rateLimited = results.filter((r) => r.status === 429).length;
+  const testedResults = results.filter((r) => !r.skipped);
+  const skippedCount = results.length - testedResults.length;
+  const mdSupported = testedResults.filter((r) => r.supported).length;
+  const mdUnsupported = testedResults.length - mdSupported;
+  const supportRate =
+    testedResults.length > 0 ? Math.round((mdSupported / testedResults.length) * 100) : 0;
+  const fetchErrors = testedResults.filter((r) => r.error).length;
+  const rateLimited = testedResults.filter((r) => r.status === 429).length;
 
   const pageLabel = wasSampled ? 'sampled pages' : 'pages';
   const suffix =
@@ -72,7 +89,8 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
 
   const details: Record<string, unknown> = {
     totalPages,
-    testedPages: results.length,
+    testedPages: testedResults.length,
+    skippedPages: skippedCount,
     sampled: wasSampled,
     mdSupported,
     mdUnsupported,
@@ -88,7 +106,7 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
       id,
       category,
       status: 'pass',
-      message: `${mdSupported}/${results.length} ${pageLabel} support .md URLs (${supportRate}%)${suffix}`,
+      message: `${mdSupported}/${testedResults.length} ${pageLabel} support .md URLs (${supportRate}%)${suffix}`,
       details,
     };
   }
@@ -98,7 +116,7 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
       id,
       category,
       status: 'warn',
-      message: `${mdSupported}/${results.length} ${pageLabel} support .md URLs (${supportRate}%); inconsistent support${suffix}`,
+      message: `${mdSupported}/${testedResults.length} ${pageLabel} support .md URLs (${supportRate}%); inconsistent support${suffix}`,
       details,
     };
   }
@@ -107,7 +125,7 @@ async function check(ctx: CheckContext): Promise<CheckResult> {
     id,
     category,
     status: 'fail',
-    message: `No ${pageLabel} support .md URLs (0/${results.length} tested)${suffix}`,
+    message: `No ${pageLabel} support .md URLs (0/${testedResults.length} tested)${suffix}`,
     details,
   };
 }

src/checks/page-size/content-start-position.ts

@@ -13,13 +13,76 @@ interface PagePositionResult {
   error?: string;
 }
 
-const CSS_PATTERN = /[{}\s]*[a-z-]+\s*:\s*[^;]+;/;
+const CSS_PATTERN = /[{}\s]*[a-z0-9_-]+\s*:\s*[^;]+;/;
 const JS_PATTERNS = [/^\s*(function|var|const|let|import|export)\b/, /^\s*\/\//, /[{};]\s*$/];
+const INLINE_SCRIPT_MIN_LENGTH = 200;
+const INLINE_SCRIPT_TOKENS =
+  /function\s*\(|=>\s*\{|document\.|window\.|localStorage|\.addEventListener|\.getElementById|\.querySelector|\.setAttribute|self\.\\/;
 const NAV_MAX_LENGTH = 40;
 
+/** Measure how much of a line is markdown link syntax: `[text](url)` or `[![img](src)](url)` */
+function linkDensity(line: string): number {
+  // Match plain links [text](url) and image links [![alt](src)](url)
+  const links = line.match(/\[(?:[^[\]]*|!\[[^\]]*\]\([^)]*\))*\]\([^)]*\)/g);
+  if (!links) return 0;
+  return links.join('').length / line.length;
+}
+
+/** Returns true if the line is script or CSS content that should be ignored. */
+function isBoilerplateLine(line: string): boolean {
+  if (CSS_PATTERN.test(line)) return true;
+  if (JS_PATTERNS.some((p) => p.test(line))) return true;
+  if (line.length >= INLINE_SCRIPT_MIN_LENGTH && INLINE_SCRIPT_TOKENS.test(line)) return true;
+  return false;
+}
+
+/**
+ * Check whether a heading is followed by prose (content heading) rather than
+ * a list of links (sidebar/nav heading). Looks ahead up to 6 non-empty,
+ * non-boilerplate lines after the heading for a prose paragraph.
+ */
+function headingFollowedByContent(lines: string[], headingIdx: number): boolean {
+  // Skip the heading line itself and any setext underline
+  let start = headingIdx + 1;
+  if (start < lines.length && /^[=-]+$/.test(lines[start].trim())) {
+    start++;
+  }
+
+  let nonEmptyCount = 0;
+  for (let i = start; i < lines.length && nonEmptyCount < 6; i++) {
+    const t = lines[i].trim();
+    if (t.length === 0) continue;
+
+    // Skip script/CSS boilerplate — don't count it as "content after heading"
+    if (isBoilerplateLine(t)) continue;
+
+    nonEmptyCount++;
+
+    // If we hit a link-heavy line or a list item starting with [, keep scanning
+    if (linkDensity(t) > 0.5) continue;
+    if (/^\*\s+\[/.test(t)) continue;
+    if (/^\]\(/.test(t)) continue;
+
+    // Another heading (ATX or setext) means the previous one had no prose body
+    if (/^#{1,6}\s/.test(t)) return false;
+    const nextLine = i + 1 < lines.length ? lines[i + 1].trim() : '';
+    if (/^[=-]+$/.test(nextLine) && nextLine.length >= 2) return false;
+
+    // A line > NAV_MAX_LENGTH that isn't a link is likely real prose
+    if (t.length > NAV_MAX_LENGTH && linkDensity(t) < 0.5) return true;
+
+    // A shorter line with sentence-ending punctuation is also prose
+    if (/[.!?]$/.test(t) && t.length >= 10 && linkDensity(t) < 0.5) return true;
+
+    // Short lines under headings are typically nav items
+  }
+  return false;
+}
+
 /**
  * Find the character position where meaningful content begins in converted markdown.
- * Meaningful content is a heading or a prose paragraph (not CSS, JS, or short nav text).
+ * Meaningful content is a heading followed by prose, or a standalone prose paragraph
+ * that isn't navigation, scripts, CSS, or link-heavy boilerplate.
  */
 function findContentStart(markdown: string): number {
   const lines = markdown.split('\n');
@@ -34,37 +97,52 @@ function findContentStart(markdown: string): number {
       continue;
     }
 
-    // ATX heading: starts with # at beginning of line
-    if (/^#{1,6}\s+\S/.test(trimmed)) {
-      return charPos;
+    // ATX heading h1-h4 followed by prose content (not a nav/sidebar heading)
+    if (/^#{1,4}\s+\S/.test(trimmed) && !/^#{5,6}\s/.test(trimmed)) {
+      if (headingFollowedByContent(lines, idx)) {
+        return charPos;
+      }
+      // Otherwise skip it as a sidebar/nav heading
+      charPos += line.length + 1;
+      continue;
     }
 
-    // Setext heading: current line is text, next line is === or ---
+    // Setext heading followed by prose content
     const nextLine = idx + 1 < lines.length ? lines[idx + 1].trim() : '';
     if (/^[=-]+$/.test(nextLine) && nextLine.length >= 2 && trimmed.length > 0) {
-      return charPos;
+      if (headingFollowedByContent(lines, idx)) {
+        return charPos;
+      }
+      charPos += line.length + 1;
+      continue;
     }
 
-    // Skip CSS-like lines
-    if (CSS_PATTERN.test(trimmed)) {
+    // Skip CSS, JS, and inline script boilerplate
+    if (isBoilerplateLine(trimmed)) {
       charPos += line.length + 1;
       continue;
     }
 
-    // Skip JS-like lines
-    if (JS_PATTERNS.some((p) => p.test(trimmed))) {
+    // Skip lines dominated by markdown link syntax (nav bars, TOC, link lists)
+    if (linkDensity(trimmed) > 0.5) {
       charPos += line.length + 1;
       continue;
     }
 
-    // Skip very short nav-like tokens (e.g., "Home", "Docs", "API")
-    if (trimmed.length <= NAV_MAX_LENGTH && !/[.!?]/.test(trimmed) && !trimmed.includes(' ')) {
+    // Skip bare link fragments from Turndown splitting links across lines: `](/path)`
+    if (/^\]\(/.test(trimmed)) {
       charPos += line.length + 1;
       continue;
     }
 
-    // Prose-like paragraph: contains spaces (multiple words) and is reasonably long
-    if (trimmed.length > NAV_MAX_LENGTH || (trimmed.includes(' ') && trimmed.length > 20)) {
+    // Standalone prose paragraph (not preceded by a heading we recognized).
+    // Must be a strong signal of real content to avoid matching UI chrome like
+    // "Press Enter to activate dropdown" or "Select language: English".
+    // Require sentence punctuation + substantial length, or very long text.
+    if (trimmed.length >= 80 && linkDensity(trimmed) < 0.5) {
+      return charPos;
+    }
+    if (/[.!?]$/.test(trimmed) && trimmed.length >= 40 && linkDensity(trimmed) < 0.5) {
       return charPos;
     }
 

src/cli/formatters/text.ts

@@ -69,18 +69,50 @@ const DETAIL_FORMATTERS: Record<string, DetailFormatter> = {
     const pages = details.pageResults as PageResult[] | undefined;
     if (!pages) return [];
     return pages
-      .filter((p) => !p.supported)
-      .map((p) => formatDetailLine('warn', p.url, 'no .md URL found'));
+      .filter((p) => !p.supported && !p.skipped)
+      .map((p) => {
+        const msg = p.alreadyMd ? '.md URL serves HTML, not markdown' : 'no .md URL found';
+        return formatDetailLine('warn', p.url, msg);
+      });
   },
 
   'content-negotiation': (details) => {
     const pages = details.pageResults as PageResult[] | undefined;
     if (!pages) return [];
     return pages
-      .filter((p) => p.status !== 'pass')
+      .filter((p) => p.classification !== 'markdown-with-correct-type' && !p.skipped)
       .map((p) => {
-        const classification = (p.classification as string) ?? '';
-        return formatDetailLine(p.status, p.url, classification);
+        const status = p.classification === 'markdown-with-wrong-type' ? 'warn' : 'fail';
+        const urlIsMd = /\.mdx?$/i.test(new URL(p.url).pathname);
+        let label: string;
+        if (p.classification === 'markdown-with-wrong-type') {
+          const ct = (p.contentType as string) || 'unknown';
+          label = `returns markdown but content-type is ${ct}`;
+        } else if (urlIsMd) {
+          label = '.md URL serves HTML, not markdown';
+        } else {
+          label = 'returns HTML, ignores Accept header';
+        }
+        return formatDetailLine(status, p.url, label);
+      });
+  },
+
+  'markdown-code-fence-validity': (details) => {
+    const pages = details.pageResults as PageResult[] | undefined;
+    if (!pages) return [];
+    return pages
+      .filter((p) => p.status !== 'pass')
+      .flatMap((p) => {
+        const issues =
+          (p.issues as Array<{ line: number; type: string; opener: string; closer?: string }>) ??
+          [];
+        return issues.map((issue) => {
+          const info =
+            issue.type === 'unclosed'
+              ? `unclosed ${issue.opener} at line ${issue.line}`
+              : `${issue.opener} closed with ${issue.closer} at line ${issue.line}`;
+          return formatDetailLine(issue.type === 'unclosed' ? 'fail' : 'warn', p.url, info);
+        });
       });
   },