feat(cli): add --scrape-mode option to fetch-url command

Author: arabold

README.md

@@ -267,7 +267,8 @@ docs-cli fetch-url <url> [options]
 
 **Options:**
 
-- `--no-follow-redirects`: Disable following HTTP redirects (default: follow redirects)
+- `--no-follow-redirects`: Disable following HTTP redirects (default: follow redirects).
+- `--scrape-mode <mode>`: HTML processing strategy: 'fetch' (fast, less JS), 'playwright' (slow, full JS), 'auto' (default).
 
 **Examples:**
 

src/cli.ts

@@ -228,10 +228,26 @@ async function main() {
         "--no-follow-redirects",
         "Disable following HTTP redirects (default: follow redirects)",
       )
+      .option(
+        "--scrape-mode <mode>",
+        "HTML processing strategy: 'fetch' (fast, less JS), 'playwright' (slow, full JS), 'auto' (default)",
+        (value) => {
+          const validModes = ["fetch", "playwright", "auto"];
+          if (!validModes.includes(value)) {
+            console.warn(
+              `Warning: Invalid scrape mode '${value}'. Using default 'auto'.`,
+            );
+            return "auto";
+          }
+          return value;
+        },
+        "auto",
+      )
       .action(async (url, options) => {
         const content = await tools.fetchUrl.execute({
           url,
           followRedirects: options.followRedirects,
+          scrapeMode: options.scrapeMode, // Pass the scrapeMode option
         });
         console.log(content);
       });

src/tools/FetchUrlTool.ts

@@ -6,9 +6,9 @@ import type {
 } from "../scraper/fetcher";
 import { ContentProcessingPipeline } from "../scraper/middleware/ContentProcessorPipeline";
 import {
-  HtmlDomParserMiddleware,
   HtmlMetadataExtractorMiddleware,
   HtmlSanitizerMiddleware,
+  HtmlSelectProcessorMiddleware, // Import the new middleware
   HtmlToMarkdownMiddleware,
   MarkdownMetadataExtractorMiddleware,
 } from "../scraper/middleware/components";
@@ -30,6 +30,15 @@ export interface FetchUrlToolOptions {
    * @default true
    */
   followRedirects?: boolean;
+
+  /**
+   * Determines the HTML processing strategy.
+   * - 'fetch': Use a simple DOM parser (faster, less JS support).
+   * - 'playwright': Use a headless browser (slower, full JS support).
+   * - 'auto': Automatically select the best strategy (currently defaults to 'playwright').
+   * @default 'auto'
+   */
+  scrapeMode?: "fetch" | "playwright" | "auto";
 }
 
 /**
@@ -57,7 +66,7 @@ export class FetchUrlTool {
    * @throws {ToolError} If fetching or processing fails
    */
   async execute(options: FetchUrlToolOptions): Promise<string> {
-    const { url } = options;
+    const { url, scrapeMode = "auto" } = options; // Destructure scrapeMode with default
 
     // Check all fetchers first (helpful for testing and future extensions)
     const canFetchResults = this.fetchers.map((f) => f.canFetch(url));
@@ -104,17 +113,18 @@ export class FetchUrlTool {
           followRedirects: options.followRedirects ?? true,
           excludeSelectors: undefined, // Not currently configurable via this tool
           ignoreErrors: false,
+          scrapeMode: scrapeMode, // Pass the scrapeMode
         } satisfies ScraperOptions,
       };
 
       let pipeline: ContentProcessingPipeline;
       if (initialContext.contentType.startsWith("text/html")) {
         // Updated HTML pipeline for FetchUrlTool
         pipeline = new ContentProcessingPipeline([
-          new HtmlDomParserMiddleware(),
+          new HtmlSelectProcessorMiddleware(), // Use the selector middleware
           new HtmlMetadataExtractorMiddleware(), // Keep for potential future use, though title isn't returned
           // No Link Extractor needed
-          new HtmlSanitizerMiddleware(), // Renamed instantiation, use default selectors
+          new HtmlSanitizerMiddleware(), // Use default selectors
           new HtmlToMarkdownMiddleware(),
         ]);
       } else if (