feat: add fetch-url tool to CLI and MCP server

- Add FetchUrlTool supporting both HTTP and local file URLs
- Implement fetch-url CLI command with --no-follow-redirects flag
- Add fetch_url MCP tool with followRedirects option
- Add comprehensive tests covering HTTP and file URL scenarios
- Update documentation in README.md and ARCHITECTURE.md

Closes #34

Author: arabold

ARCHITECTURE.md

@@ -212,7 +212,7 @@ This design allows easy addition of new embedding providers while maintaining co
 
 **Database Location:** The application determines the database file (`documents.db`) location dynamically:
 
-1. It first checks for a `.store` directory in the current working directory (`process.cwd()`). If `.store/documents.db` exists, it uses this path. This prioritizes local development databases.
+1. It first checks for a `.store` directory in the current project directory. If `.store/documents.db` exists, it uses this path. This prioritizes local development databases.
 2. If the local `.store/documents.db` does not exist, it defaults to a standard, OS-specific application data directory (e.g., `~/Library/Application Support/docs-mcp-server/` on macOS, `~/.local/share/docs-mcp-server/` on Linux) determined using the `env-paths` library. This ensures a stable, persistent location when running via `npx` or outside a local project context.
 
 Documents are stored with URLs and sequential ordering to maintain source context:

README.md

@@ -25,6 +25,7 @@ The server exposes MCP tools for:
 - Listing indexed libraries (`list_libraries`).
 - Finding appropriate versions (`find_version`).
 - Removing indexed documents (`remove_docs`).
+- Fetching single URLs (`fetch_url`): Fetches a URL and returns its content as Markdown.
 
 ## Configuration
 
@@ -250,11 +251,31 @@ npx -y --package=@arabold/docs-mcp-server docs-cli --help
 ```bash
 docs-cli scrape --help
 docs-cli search --help
+docs-cli fetch-url --help
 docs-cli find-version --help
 docs-cli remove --help
 docs-cli list --help
 ```
 
+### Fetching Single URLs (`fetch-url`)
+
+Fetches a single URL and converts its content to Markdown. Unlike `scrape`, this command does not crawl links or store the content.
+
+```bash
+docs-cli fetch-url <url> [options]
+```
+
+**Options:**
+
+- `--no-follow-redirects`: Disable following HTTP redirects (default: follow redirects)
+
+**Examples:**
+
+```bash
+# Fetch a URL and convert to Markdown
+docs-cli fetch-url https://example.com/page.html
+```
+
 ### Scraping Documentation (`scrape`)
 
 Scrapes and indexes documentation from a given URL for a specific library.

src/cli.ts

@@ -3,8 +3,16 @@ import "dotenv/config";
 import { Command } from "commander";
 import packageJson from "../package.json";
 import { PipelineManager } from "./pipeline/PipelineManager";
+import { FileFetcher, HttpFetcher } from "./scraper/fetcher";
+import { HtmlProcessor } from "./scraper/processor";
 import { DocumentManagementService } from "./store/DocumentManagementService";
-import { FindVersionTool, ListLibrariesTool, ScrapeTool, SearchTool } from "./tools";
+import {
+  FetchUrlTool,
+  FindVersionTool,
+  ListLibrariesTool,
+  ScrapeTool,
+  SearchTool,
+} from "./tools";
 import { LogLevel, setLogLevel } from "./utils/logger";
 
 const formatOutput = (data: unknown) => JSON.stringify(data, null, 2);
@@ -27,6 +35,11 @@ async function main() {
       findVersion: new FindVersionTool(docService),
       scrape: new ScrapeTool(docService, pipelineManager), // Pass manager
       search: new SearchTool(docService),
+      fetchUrl: new FetchUrlTool(
+        new HttpFetcher(),
+        new FileFetcher(),
+        new HtmlProcessor(),
+      ),
     };
 
     const program = new Command();
@@ -184,6 +197,21 @@ async function main() {
         }
       });
 
+    program
+      .command("fetch-url <url>")
+      .description("Fetch a URL and convert its content to Markdown")
+      .option(
+        "--no-follow-redirects",
+        "Disable following HTTP redirects (default: follow redirects)",
+      )
+      .action(async (url, options) => {
+        const content = await tools.fetchUrl.execute({
+          url,
+          followRedirects: options.followRedirects,
+        });
+        console.log(content);
+      });
+
     // Hook to set log level after parsing global options but before executing command action
     program.hook("preAction", (thisCommand) => {
       // Global options are attached to the program (thisCommand)

src/mcp/index.ts

@@ -5,9 +5,12 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
 import { z } from "zod";
 import { PipelineManager } from "../pipeline/PipelineManager";
 import { PipelineJobStatus } from "../pipeline/types";
+import { FileFetcher, HttpFetcher } from "../scraper/fetcher";
+import { HtmlProcessor } from "../scraper/processor";
 import { DocumentManagementService } from "../store/DocumentManagementService";
 import {
   CancelJobTool,
+  FetchUrlTool,
   FindVersionTool,
   GetJobInfoTool,
   ListJobsTool,
@@ -39,15 +42,17 @@ export async function startServer() {
     const tools = {
       listLibraries: new ListLibrariesTool(docService),
       findVersion: new FindVersionTool(docService),
-      // TODO: Update ScrapeTool constructor if needed to accept PipelineManager
-      // ScrapeTool currently uses docService.getPipelineManager() which doesn't exist.
-      // Pass both docService and pipelineManager to ScrapeTool constructor
       scrape: new ScrapeTool(docService, pipelineManager),
       search: new SearchTool(docService),
       listJobs: new ListJobsTool(pipelineManager),
       getJobInfo: new GetJobInfoTool(pipelineManager),
       cancelJob: new CancelJobTool(pipelineManager),
-      remove: new RemoveTool(docService), // Instantiate RemoveTool
+      remove: new RemoveTool(docService),
+      fetchUrl: new FetchUrlTool(
+        new HttpFetcher(),
+        new FileFetcher(),
+        new HtmlProcessor(),
+      ),
     };
 
     const server = new McpServer(
@@ -305,6 +310,30 @@ ${formattedResults.join("")}`,
       },
     );
 
+    // Fetch URL tool
+    server.tool(
+      "fetch_url",
+      "Fetch a single URL and convert its content to Markdown",
+      {
+        url: z.string().url().describe("The URL to fetch and convert to markdown"),
+        followRedirects: z
+          .boolean()
+          .optional()
+          .default(true)
+          .describe("Whether to follow HTTP redirects (3xx responses)"),
+      },
+      async ({ url, followRedirects }) => {
+        try {
+          const result = await tools.fetchUrl.execute({ url, followRedirects });
+          return createResponse(result);
+        } catch (error) {
+          return createError(
+            `Failed to fetch URL: ${error instanceof Error ? error.message : String(error)}`,
+          );
+        }
+      },
+    );
+
     // Cancel job tool
     server.tool(
       "cancel_job",

src/scraper/processor/types.ts

@@ -1,5 +1,7 @@
 import type { RawContent } from "../fetcher/types";
 
+export type { RawContent };
+
 /**
  * Processed content in normalized form (markdown)
  * with extracted metadata and references