feat(scraper): replace JSDOM with Cheerio for HTML parsing

Author: arabold

package-lock.json

@@ -44,8 +44,8 @@
     "axios": "^1.8.3",
     "axios-retry": "^4.5.0",
     "better-sqlite3": "^11.9.1",
+    "cheerio": "^1.0.0",
     "commander": "^13.1.0",
-    "dompurify": "^3.2.4",
     "dotenv": "^16.4.7",
     "env-paths": "^3.0.0",
     "fuse.js": "^7.1.0",

package.json

@@ -29,7 +29,7 @@ export class PipelineWorker {
     const { id: jobId, library, version, options, abortController } = job;
     const signal = abortController.signal;
 
-    logger.info(`[${jobId}] Worker starting job for ${library}@${version}`);
+    logger.debug(`[${jobId}] Worker starting job for ${library}@${version}`);
 
     try {
       // --- Core Job Logic ---

src/pipeline/PipelineWorker.ts

@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
 import type { ScraperOptions } from "../types";
 import { ContentProcessingPipeline } from "./ContentProcessorPipeline";
 import {
-  HtmlDomParserMiddleware,
+  HtmlCheerioParserMiddleware, // Updated import
   HtmlLinkExtractorMiddleware,
   HtmlMetadataExtractorMiddleware,
   HtmlSanitizerMiddleware,
@@ -55,10 +55,10 @@ const createPipelineTestContext = (
 
 // Define the standard HTML pipeline for tests
 const htmlPipeline = new ContentProcessingPipeline([
-  new HtmlDomParserMiddleware(),
+  new HtmlCheerioParserMiddleware(),
   new HtmlMetadataExtractorMiddleware(),
   new HtmlLinkExtractorMiddleware(),
-  new HtmlSanitizerMiddleware(), // Uses default selectors if none provided in context.options
+  new HtmlSanitizerMiddleware(),
   new HtmlToMarkdownMiddleware(),
 ]);
 

src/scraper/middleware/ContentProcessorPipeline.test.ts

@@ -0,0 +1,49 @@
+import * as cheerio from "cheerio";
+import { logger } from "../../../utils/logger";
+import type { ContentProcessingContext, ContentProcessorMiddleware } from "../types";
+
+/**
+ * Middleware to parse HTML string/buffer content into a Cheerio object.
+ * It populates the `context.dom` property.
+ * Assumes the input HTML in `context.content` is the final version to be parsed
+ * (e.g., after potential rendering by Playwright or modification by JS execution).
+ */
+export class HtmlCheerioParserMiddleware implements ContentProcessorMiddleware {
+  async process(
+    context: ContentProcessingContext,
+    next: () => Promise<void>,
+  ): Promise<void> {
+    // Only process HTML content
+    if (!context.contentType.startsWith("text/html")) {
+      await next();
+      return;
+    }
+
+    // Ensure content is a string for Cheerio
+    const htmlString =
+      typeof context.content === "string"
+        ? context.content
+        : Buffer.from(context.content).toString("utf-8");
+
+    try {
+      logger.debug(`Parsing HTML content with Cheerio from ${context.source}`);
+      // Load the HTML string using Cheerio
+      const $ = cheerio.load(htmlString);
+
+      // Add the Cheerio API object to the context
+      context.dom = $;
+
+      // Proceed to the next middleware
+      await next();
+    } catch (error) {
+      logger.error(`Failed to parse HTML with Cheerio for ${context.source}: ${error}`);
+      context.errors.push(
+        error instanceof Error
+          ? error
+          : new Error(`Cheerio HTML parsing failed: ${String(error)}`),
+      );
+      // Do not proceed further down the pipeline if parsing fails
+      return;
+    }
+  }
+}