fix: TXT and MD chunking

Author: fsender

README.md

@@ -516,7 +516,10 @@ Zero-config by default (uses `auto` mode). Customize in `.opencode/codebase-inde
     "autoGc": true,                           // Auto garbage collection
     "gcIntervalDays": 7,                      // GC interval (days)
     "gcOrphanThreshold": 100,                 // GC trigger threshold
-    "requireProjectMarker": true              // Require .git/package.json to index
+    "requireProjectMarker": true,             // Require .git/package.json to index
+    "maxDepth": 5,                            // Max directory depth (-1=unlimited, 0=root only)
+    "maxFilesPerDirectory": 100,              // Max files per directory (smallest first)
+    "fallbackToTextOnMaxChunks": true         // Fallback to text chunking on maxChunksPerFile
   },
 
   // === Search ===
@@ -590,6 +593,9 @@ String values in `codebase-index.json` can reference environment variables with
 | `gcIntervalDays` | `7` | Run GC on initialization if last GC was more than N days ago |
 | `gcOrphanThreshold` | `100` | Run GC after indexing if orphan count exceeds this threshold |
 | `requireProjectMarker` | `true` | Require a project marker (`.git`, `package.json`, etc.) to enable file watching and auto-indexing. Prevents accidentally indexing large directories like home. Set to `false` to index any directory. |
+| `maxDepth` | `5` | Max directory traversal depth. `-1` = unlimited, `0` = only files in root dir, `1` = one level of subdirectories, etc. |
+| `maxFilesPerDirectory` | `100` | Max files to index per directory. Always picks the smallest files first. |
+| `fallbackToTextOnMaxChunks` | `true` | When a file exceeds `maxChunksPerFile`, fallback to text-based (line-by-line) chunking instead of skipping the rest of the file. |
 | **search** | | |
 | `maxResults` | `20` | Maximum results to return |
 | `minScore` | `0.1` | Minimum similarity score (0-1). Lower = more results |

native/src/lib.rs

@@ -25,6 +25,12 @@ pub fn parse_file(file_path: String, content: String) -> Result<Vec<CodeChunk>>
     parser::parse_file_internal(&file_path, &content).map_err(|e| Error::from_reason(e.to_string()))
 }
 
+#[napi]
+pub fn parse_file_as_text(file_path: String, content: String) -> Result<Vec<CodeChunk>> {
+    parser::parse_file_as_text_internal(&file_path, &content)
+        .map_err(|e| Error::from_reason(e.to_string()))
+}
+
 #[napi]
 pub fn parse_files(files: Vec<FileInput>) -> Result<Vec<ParsedFile>> {
     parser::parse_files_parallel(files).map_err(|e| Error::from_reason(e.to_string()))

native/src/parser.rs

@@ -58,6 +58,16 @@ pub fn parse_file_internal(file_path: &str, content: &str) -> Result<Vec<CodeChu
     extract_chunks(&tree, content, &language)
 }
 
+pub fn parse_file_as_text_internal(file_path: &str, content: &str) -> Result<Vec<CodeChunk>> {
+    let ext = Path::new(file_path)
+        .extension()
+        .and_then(|e| e.to_str())
+        .unwrap_or("");
+
+    let language = Language::from_extension(ext);
+    Ok(chunk_by_lines(content, &language))
+}
+
 pub fn parse_files_parallel(files: Vec<FileInput>) -> Result<Vec<ParsedFile>> {
     let results: Vec<ParsedFile> = files
         .par_iter()

package-lock.json

@@ -70,6 +70,7 @@
     ]
   },
   "dependencies": {
+    "@opencode-ai/plugin": "~1.3.13",
     "chokidar": "^5.0.0",
     "ignore": "^7.0.5",
     "p-queue": "^9.1.1",
@@ -80,7 +81,6 @@
     "@eslint/js": "^9.39.4",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@napi-rs/cli": "^3.6.0",
-    "@opencode-ai/plugin": "^1.3.13",
     "@types/node": "^25.5.2",
     "@vitest/coverage-v8": "^4.1.2",
     "eslint": "^9.39.4",
@@ -106,4 +106,4 @@
       "optional": true
     }
   }
-}
+}

package.json

@@ -16,13 +16,26 @@ export interface IndexingConfig {
   autoGc: boolean;
   gcIntervalDays: number;
   gcOrphanThreshold: number;
-  /** 
-   * When true (default), requires a project marker (.git, package.json, Cargo.toml, etc.) 
+  /**
+   * When true (default), requires a project marker (.git, package.json, Cargo.toml, etc.)
    * to be present before enabling file watching and auto-indexing.
-   * This prevents accidentally watching/indexing large non-project directories like home.
-   * Set to false to allow indexing any directory.
    */
   requireProjectMarker: boolean;
+  /**
+   * Max directory traversal depth. -1 = unlimited, 0 = only files in the root dir,
+   * 1 = one level of subdirectories, etc. Default: 5
+   */
+  maxDepth: number;
+  /**
+   * Max number of files to index per directory. Always picks the smallest files first.
+   * Default: 100
+   */
+  maxFilesPerDirectory: number;
+  /**
+   * When a file hits maxChunksPerFile, fallback to text-based (chunk_by_lines) parsing
+   * instead of skipping the rest of the file. Default: true
+   */
+  fallbackToTextOnMaxChunks: boolean;
 }
 
 export interface SearchConfig {
@@ -128,6 +141,9 @@ function getDefaultIndexingConfig(): IndexingConfig {
     gcIntervalDays: 7,
     gcOrphanThreshold: 100,
     requireProjectMarker: true,
+    maxDepth: 5,
+    maxFilesPerDirectory: 100,
+    fallbackToTextOnMaxChunks: true,
   };
 }
 
@@ -237,6 +253,9 @@ export function parseConfig(raw: unknown): ParsedCodebaseIndexConfig {
     gcIntervalDays: typeof rawIndexing.gcIntervalDays === "number" ? Math.max(1, rawIndexing.gcIntervalDays) : defaultIndexing.gcIntervalDays,
     gcOrphanThreshold: typeof rawIndexing.gcOrphanThreshold === "number" ? Math.max(0, rawIndexing.gcOrphanThreshold) : defaultIndexing.gcOrphanThreshold,
     requireProjectMarker: typeof rawIndexing.requireProjectMarker === "boolean" ? rawIndexing.requireProjectMarker : defaultIndexing.requireProjectMarker,
+    maxDepth: typeof rawIndexing.maxDepth === "number" ? (rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth) : defaultIndexing.maxDepth,
+    maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory,
+    fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks,
   };
 
   const rawSearch = (input.search && typeof input.search === "object" ? input.search : {}) as Record<string, unknown>;

src/config/schema.ts

@@ -29,6 +29,7 @@ import {
   hashFile,
   hashContent,
   extractCalls,
+  parseFileAsText,
 } from "../native/index.js";
 import type { SymbolData, CallEdgeData } from "../native/index.js";
 import { getBranchOrDefault, getBaseBranch, isGitRepo } from "../git/index.js";
@@ -1736,7 +1737,8 @@ export class Indexer {
       includePatterns,
       this.config.exclude,
       this.config.indexing.maxFileSize,
-      this.config.knowledgeBases
+      this.config.knowledgeBases,
+      { maxDepth: this.config.indexing.maxDepth, maxFilesPerDirectory: this.config.indexing.maxFilesPerDirectory }
     );
 
     return createCostEstimate(files, configuredProviderInfo);
@@ -1786,7 +1788,8 @@ export class Indexer {
       includePatterns,
       this.config.exclude,
       this.config.indexing.maxFileSize,
-      this.config.knowledgeBases
+      this.config.knowledgeBases,
+      { maxDepth: this.config.indexing.maxDepth, maxFilesPerDirectory: this.config.indexing.maxFilesPerDirectory }
     );
 
     stats.totalFiles = files.length;
@@ -1871,7 +1874,17 @@ export class Indexer {
       }
 
       let fileChunkCount = 0;
-      for (const chunk of parsed.chunks) {
+      let chunksToProcess = parsed.chunks;
+
+      if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
+        const changedFile = changedFiles.find(f => f.path === parsed.path);
+        if (changedFile) {
+          const textChunks = parseFileAsText(parsed.path, changedFile.content);
+          chunksToProcess = textChunks;
+        }
+      }
+
+      for (const chunk of chunksToProcess) {
         if (fileChunkCount >= this.config.indexing.maxChunksPerFile) {
           break;
         }

src/indexer/index.ts

@@ -178,6 +178,11 @@ export function parseFile(filePath: string, content: string): CodeChunk[] {
   return result.map(mapChunk);
 }
 
+export function parseFileAsText(filePath: string, content: string): CodeChunk[] {
+  const result = native.parseFileAsText(filePath, content);
+  return result.map(mapChunk);
+}
+
 export function parseFiles(files: FileInput[]): ParsedFile[] {
   const result = native.parseFiles(files);
   return result.map((f: any) => ({

src/native/index.ts

@@ -127,17 +127,27 @@ function matchGlob(filePath: string, pattern: string): boolean {
   return regex.test(filePath);
 }
 
+export interface WalkOptions {
+  maxDepth: number;
+  maxFilesPerDirectory: number;
+}
+
 export async function* walkDirectory(
   dir: string,
   projectRoot: string,
   includePatterns: string[],
   excludePatterns: string[],
   ignoreFilter: Ignore,
   maxFileSize: number,
-  skipped: SkippedFile[]
+  skipped: SkippedFile[],
+  options: WalkOptions,
+  currentDepth: number = 0
 ): AsyncGenerator<{ path: string; size: number }> {
   const entries = await fsPromises.readdir(dir, { withFileTypes: true });
 
+  const filesInDir: Array<{ path: string; size: number }> = [];
+  const subdirs: Array<{ fullPath: string; relativePath: string }> = [];
+
   for (const entry of entries) {
     const fullPath = path.join(dir, entry.name);
     const relativePath = path.relative(projectRoot, fullPath);
@@ -164,15 +174,7 @@ export async function* walkDirectory(
     }
 
     if (entry.isDirectory()) {
-      yield* walkDirectory(
-        fullPath,
-        projectRoot,
-        includePatterns,
-        excludePatterns,
-        ignoreFilter,
-        maxFileSize,
-        skipped
-      );
+      subdirs.push({ fullPath, relativePath });
     } else if (entry.isFile()) {
       const stat = await fsPromises.stat(fullPath);
 
@@ -197,19 +199,49 @@ export async function* walkDirectory(
       }
 
       if (matched) {
-        yield { path: fullPath, size: stat.size };
+        filesInDir.push({ path: fullPath, size: stat.size });
       }
     }
   }
+
+  // Sort by size ascending, keep only the smallest maxFilesPerDirectory files
+  filesInDir.sort((a, b) => a.size - b.size);
+  const limitedFiles = filesInDir.slice(0, options.maxFilesPerDirectory);
+  for (const f of limitedFiles) {
+    yield f;
+  }
+  for (let i = options.maxFilesPerDirectory; i < filesInDir.length; i++) {
+    skipped.push({ path: path.relative(projectRoot, filesInDir[i].path), reason: "excluded" });
+  }
+
+  // Recurse into subdirectories respecting depth limit
+  const canRecurse = options.maxDepth === -1 || currentDepth < options.maxDepth;
+  if (canRecurse) {
+    for (const sub of subdirs) {
+      yield* walkDirectory(
+        sub.fullPath,
+        projectRoot,
+        includePatterns,
+        excludePatterns,
+        ignoreFilter,
+        maxFileSize,
+        skipped,
+        options,
+        currentDepth + 1
+      );
+    }
+  }
 }
 
 export async function collectFiles(
   projectRoot: string,
   includePatterns: string[],
   excludePatterns: string[],
   maxFileSize: number,
-  additionalRoots?: string[]
+  additionalRoots?: string[],
+  walkOptions?: WalkOptions
 ): Promise<CollectFilesResult> {
+  const opts: WalkOptions = walkOptions ?? { maxDepth: 5, maxFilesPerDirectory: 100 };
   const ignoreFilter = createIgnoreFilter(projectRoot);
   const files: Array<{ path: string; size: number }> = [];
   const skipped: SkippedFile[] = [];
@@ -222,7 +254,9 @@ export async function collectFiles(
     excludePatterns,
     ignoreFilter,
     maxFileSize,
-    skipped
+    skipped,
+    opts,
+    0
   )) {
     files.push(file);
   }
@@ -253,7 +287,9 @@ export async function collectFiles(
           excludePatterns,
           kbIgnoreFilter,
           maxFileSize,
-          skipped
+          skipped,
+          opts,
+          0
         )) {
           files.push(file);
         }