feat: enhance web scraping and error handling

- Add error handling to BaseScraperStrategy
- Improve HTML processing with additional selectors and link extraction
- Update CLI with ignoreErrors option
- Add tests for BaseScraperStrategy

Author: arabold

src/cli.ts

@@ -4,7 +4,12 @@ import os from "node:os";
 import path from "node:path";
 import { Command } from "commander";
 import { VectorStoreService } from "./store/VectorStoreService";
-import { FindVersionTool, ListLibrariesTool, ScrapeTool, SearchTool } from "./tools";
+import {
+  FindVersionTool,
+  ListLibrariesTool,
+  ScrapeTool,
+  SearchTool,
+} from "./tools";
 
 const formatOutput = (data: unknown) => JSON.stringify(data, null, 2);
 
@@ -39,12 +44,12 @@ async function main() {
       .description("Scrape and index documentation from a URL")
       .option("-p, --max-pages <number>", "Maximum pages to scrape", "100")
       .option("-d, --max-depth <number>", "Maximum navigation depth", "3")
-      .option("-c, --max-concurrency <number>", "Maximum concurrent page requests", "3")
       .option(
-        "--subpages-only",
-        "Allow scraping pages outside the initial URL path",
-        true,
+        "-c, --max-concurrency <number>",
+        "Maximum concurrent page requests",
+        "3"
       )
+      .option("--ignore-errors", "Ignore errors during scraping", true)
       .action(async (library, version, url, options) => {
         const result = await tools.scrape.execute({
           url,
@@ -54,6 +59,7 @@ async function main() {
             maxPages: Number.parseInt(options.maxPages),
             maxDepth: Number.parseInt(options.maxDepth),
             maxConcurrency: Number.parseInt(options.maxConcurrency),
+            ignoreErrors: options.ignoreErrors,
           },
         });
         console.log(`✅ Successfully scraped ${result.pagesScraped} pages`);
@@ -66,13 +72,13 @@ async function main() {
           "  - search react 18.0.0 'hooks' -> matches docs for React 18.0.0 or earlier versions\n" +
           "  - search react 18.0.0 'hooks' --exact-match -> only matches React 18.0.0\n" +
           "  - search typescript 5.x 'types' -> matches any TypeScript 5.x.x version\n" +
-          "  - search typescript 5.2.x 'types' -> matches any TypeScript 5.2.x version",
+          "  - search typescript 5.2.x 'types' -> matches any TypeScript 5.2.x version"
       )
       .option("-l, --limit <number>", "Maximum number of results", "5")
       .option(
         "-e, --exact-match",
         "Only use exact version match (e.g., '18.0.0' matches only 18.0.0, not 17.x.x) (default: false)",
-        false,
+        false
       )
       .action(async (library, version, query, options) => {
         const result = await tools.search.execute({
@@ -109,7 +115,10 @@ async function main() {
 
     await program.parseAsync();
   } catch (error) {
-    console.error("Error:", error instanceof Error ? error.message : String(error));
+    console.error(
+      "Error:",
+      error instanceof Error ? error.message : String(error)
+    );
     await storeService.shutdown();
     process.exit(1);
   }

src/scraper/ScraperService.test.ts

@@ -27,12 +27,13 @@ describe("ScraperService", () => {
       maxPages: 10,
       maxDepth: 1,
     };
+    const progressCallback: ProgressCallback<ScraperProgress> = vi.fn();
 
     mockRegistry.getStrategy.mockReturnValue(mockStrategy);
-    await service.scrape(options);
+    await service.scrape(options, progressCallback);
 
     expect(mockRegistry.getStrategy).toHaveBeenCalledWith(options.url);
-    expect(mockStrategy.scrape).toHaveBeenCalledWith(options, undefined);
+    expect(mockStrategy.scrape).toHaveBeenCalledWith(options, progressCallback);
   });
 
   it("should pass progress callback to strategy", async () => {
@@ -65,12 +66,13 @@ describe("ScraperService", () => {
       maxPages: 10,
       maxDepth: 1,
     };
+    const progressCallback: ProgressCallback<ScraperProgress> = vi.fn();
 
     mockRegistry.getStrategy.mockReturnValue(mockStrategy);
-    await service.scrape(options);
+    await service.scrape(options, progressCallback);
 
     expect(mockRegistry.getStrategy).toHaveBeenCalledWith(options.url);
-    expect(mockStrategy.scrape).toHaveBeenCalledWith(options, undefined);
+    expect(mockStrategy.scrape).toHaveBeenCalledWith(options, progressCallback);
   });
 
   it("should throw error if no strategy found", async () => {
@@ -84,11 +86,14 @@ describe("ScraperService", () => {
       maxPages: 10,
       maxDepth: 1,
     };
+    const progressCallback: ProgressCallback<ScraperProgress> = vi.fn();
 
     mockRegistry.getStrategy.mockReturnValue(null);
 
-    await expect(service.scrape(options)).rejects.toThrow(ScraperError);
-    await expect(service.scrape(options)).rejects.toThrow(
+    await expect(service.scrape(options, progressCallback)).rejects.toThrow(
+      ScraperError
+    );
+    await expect(service.scrape(options, progressCallback)).rejects.toThrow(
       "No scraper strategy found for URL: unknown://example.com"
     );
   });
@@ -104,10 +109,13 @@ describe("ScraperService", () => {
       maxPages: 10,
       maxDepth: 1,
     };
+    const progressCallback: ProgressCallback<ScraperProgress> = vi.fn();
 
     mockRegistry.getStrategy.mockReturnValue(mockStrategy);
     mockStrategy.scrape.mockRejectedValue(new Error("Strategy error"));
 
-    await expect(service.scrape(options)).rejects.toThrow("Strategy error");
+    await expect(service.scrape(options, progressCallback)).rejects.toThrow(
+      "Strategy error"
+    );
   });
 });

src/scraper/ScraperService.ts

@@ -20,12 +20,15 @@ export class ScraperService {
    */
   async scrape(
     options: ScraperOptions,
-    progressCallback?: ProgressCallback<ScraperProgress>,
+    progressCallback: ProgressCallback<ScraperProgress>
   ): Promise<void> {
     // Find strategy for this URL
     const strategy = this.registry.getStrategy(options.url);
     if (!strategy) {
-      throw new ScraperError(`No scraper strategy found for URL: ${options.url}`, false);
+      throw new ScraperError(
+        `No scraper strategy found for URL: ${options.url}`,
+        false
+      );
     }
 
     await strategy.scrape(options, progressCallback);

src/scraper/processor/HtmlProcessor.test.ts

@@ -83,4 +83,103 @@ describe("HtmlProcessor", () => {
     const result = await processor.process(rawContent);
     expect(result.links).toEqual([]);
   });
+
+  it("should extract links from nav sidebar before removing tags", async () => {
+    const processor = new HtmlProcessor();
+    const rawContent: RawContent = {
+      content:
+        '<html><head><title>Test</title></head><body><nav><ul><li><a href="/home">Home</a></li><li><a href="/about">About</a></li></ul></nav><p>Other content</p></body></html>',
+      mimeType: "text/html",
+      source: "https://example.com",
+    };
+    const result = await processor.process(rawContent);
+    expect(result.links).toEqual([
+      "https://example.com/home",
+      "https://example.com/about",
+    ]);
+  });
+
+  it("should remove unwanted tags and keep text from allowed tags", async () => {
+    const processor = new HtmlProcessor();
+    const rawContent: RawContent = {
+      content:
+        "<html><head><title>Test</title></head><body><nav><ul><li><a href=\"/home\">Home</a></li></ul></nav><p>This text should remain.</p><script>alert('This should be removed');</script></body></html>",
+      mimeType: "text/html",
+      source: "https://example.com",
+    };
+    const result = await processor.process(rawContent);
+    expect(result.content).toContain("This text should remain.");
+    expect(result.content).not.toContain("Home");
+    expect(result.content).not.toContain("This should be removed");
+  });
+
+  describe("Code block language detection", () => {
+    const processor = new HtmlProcessor();
+
+    it("should detect language from highlight-source-<language> on a parent", async () => {
+      const rawContent: RawContent = {
+        content:
+          '<html><head><title>Test</title></head><body><div class="highlight-source-python"><pre><code>print("Hello")</code></pre></div></body></html>',
+        mimeType: "text/html",
+        source: "https://example.com",
+      };
+      const result = await processor.process(rawContent);
+      expect(result.content).toContain("```python");
+    });
+
+    it("should detect language from highlight-<language> on a parent", async () => {
+      const rawContent: RawContent = {
+        content:
+          '<html><head><title>Test</title></head><body><div class="highlight-javascript"><pre><code>console.log("Hello")</code></pre></div></body></html>',
+        mimeType: "text/html",
+        source: "https://example.com",
+      };
+      const result = await processor.process(rawContent);
+      expect(result.content).toContain("```javascript");
+    });
+
+    it("should detect language from language-<language> on a parent", async () => {
+      const rawContent: RawContent = {
+        content:
+          '<html><head><title>Test</title></head><body><div class="language-typescript"><pre><code>console.log("Hello")</code></pre></div></body></html>',
+        mimeType: "text/html",
+        source: "https://example.com",
+      };
+      const result = await processor.process(rawContent);
+      expect(result.content).toContain("```typescript");
+    });
+
+    it("should detect language from language-<language> on the pre tag itself", async () => {
+      const rawContent: RawContent = {
+        content:
+          '<html><head><title>Test</title></head><body><pre class="language-java"><code>System.out.println("Hello")</code></pre></body></html>',
+        mimeType: "text/html",
+        source: "https://example.com",
+      };
+      const result = await processor.process(rawContent);
+      expect(result.content).toContain("```java");
+    });
+
+    it("should default to empty language if no language class is present", async () => {
+      const rawContent: RawContent = {
+        content:
+          '<html><head><title>Test</title></head><body><pre><code>print("Hello")</code></pre></body></html>',
+        mimeType: "text/html",
+        source: "https://example.com",
+      };
+      const result = await processor.process(rawContent);
+      expect(result.content).toContain("```\n");
+    });
+
+    it("should prioritize data-language attribute", async () => {
+      const rawContent: RawContent = {
+        content:
+          '<html><head><title>Test</title></head><body><div class="highlight-source-python"><pre data-language="typescript"><code>print("Hello")</code></pre></div></body></html>',
+        mimeType: "text/html",
+        source: "https://example.com",
+      };
+      const result = await processor.process(rawContent);
+      expect(result.content).toContain("```typescript");
+    });
+  });
 });