feat: implement batch operations for embeddings and chunks in the database

Author: Helweg

AGENTS.md

@@ -1,6 +1,6 @@
 # AGENTS.md - AI Agent Guidelines for opencode-codebase-index
 
-**Generated:** 2026-01-16 | **Commit:** d02b915 | **Branch:** main
+**Generated:** 2025-01-16 | **Commit:** 9f94822 | **Branch:** main
 
 Semantic codebase indexing plugin for OpenCode. Hybrid TypeScript/Rust architecture:
 - **TypeScript** (`src/`): Plugin logic, embedding providers, OpenCode tools
@@ -90,10 +90,20 @@ skill/                    # OpenCode skill guidance
 | `parse_files` | fn | Parallel multi-file parsing |
 | `hash_content` | fn | xxhash string |
 | `hash_file` | fn | xxhash file contents |
-| `VectorStore` | class | usearch wrapper (add/search/save/load) |
-| `Database` | class | SQLite: embeddings, chunks, branches, metadata |
+| `VectorStore` | class | usearch wrapper (add/addBatch/search/save/load) |
+| `Database` | class | SQLite: embeddings, chunks, branches, metadata (includes batch methods) |
 | `InvertedIndex` | class | BM25 keyword search |
 
+### Database Batch Methods
+The `Database` class exposes batch operations for high-performance bulk inserts:
+| Method | Purpose | Speedup |
+|--------|---------|---------|
+| `upsertEmbeddingsBatch` | Batch insert embeddings in single transaction | ~1.3x |
+| `upsertChunksBatch` | Batch insert chunks in single transaction | ~12x |
+| `addChunksToBranchBatch` | Batch add chunks to branch in single transaction | ~18x |
+
+These are used by the Indexer for all bulk operations. Prefer batch methods over sequential calls.
+
 ## CONVENTIONS
 
 ### Import Rules (CRITICAL - causes runtime errors if wrong)
@@ -180,10 +190,20 @@ afterEach(() => { fs.rmSync(tempDir, { recursive: true, force: true }); });
 | File | Tests |
 |------|-------|
 | `native.test.ts` | Rust bindings: parsing, vectors, hashing |
-| `database.test.ts` | SQLite: CRUD, branches, GC |
+| `database.test.ts` | SQLite: CRUD, branches, GC, batch operations |
 | `inverted-index.test.ts` | BM25 keyword search |
 | `files.test.ts` | File collection, .gitignore |
 | `cost.test.ts` | Token estimation |
+| `watcher.test.ts` | File/git branch watching |
+| `auto-gc.test.ts` | Automatic garbage collection |
+| `git.test.ts` | Git branch detection |
+
+### Benchmarks
+```bash
+npx tsx benchmarks/run.ts   # Performance testing for native operations
+```
+
+Tests batch vs sequential performance for VectorStore and SQLite operations.
 
 ## CONFIGURATION
 

src/indexer/index.ts

@@ -297,6 +297,7 @@ export class Indexer {
 
     const allMetadata = this.store.getAllMetadata();
     const chunkIds: string[] = [];
+    const chunkDataBatch: ChunkData[] = [];
 
     for (const { key, metadata } of allMetadata) {
       const chunkData: ChunkData = {
@@ -309,11 +310,14 @@ export class Indexer {
         name: metadata.name,
         language: metadata.language,
       };
-      this.database.upsertChunk(chunkData);
+      chunkDataBatch.push(chunkData);
       chunkIds.push(key);
     }
 
-    this.database.addChunksToBranch(this.currentBranch || "default", chunkIds);
+    if (chunkDataBatch.length > 0) {
+      this.database.upsertChunksBatch(chunkDataBatch);
+    }
+    this.database.addChunksToBranchBatch(this.currentBranch || "default", chunkIds);
   }
 
   private async ensureInitialized(): Promise<{
@@ -434,6 +438,8 @@ export class Indexer {
       }
     }
 
+    const chunkDataBatch: ChunkData[] = [];
+
     for (const parsed of parsedFiles) {
       currentFilePaths.add(parsed.path);
 
@@ -466,7 +472,7 @@ export class Indexer {
           name: chunk.name,
           language: chunk.language,
         };
-        database.upsertChunk(chunkData);
+        chunkDataBatch.push(chunkData);
 
         if (existingChunks.get(id) === contentHash) {
           fileChunkCount++;
@@ -489,6 +495,10 @@ export class Indexer {
       }
     }
 
+    if (chunkDataBatch.length > 0) {
+      database.upsertChunksBatch(chunkDataBatch);
+    }
+
     let removedCount = 0;
     for (const [chunkId] of existingChunks) {
       if (!currentChunkIds.has(chunkId)) {
@@ -504,7 +514,7 @@ export class Indexer {
 
     if (pendingChunks.length === 0 && removedCount === 0) {
       database.clearBranch(this.currentBranch);
-      database.addChunksToBranch(this.currentBranch, Array.from(currentChunkIds));
+      database.addChunksToBranchBatch(this.currentBranch, Array.from(currentChunkIds));
       this.fileHashCache = currentFileHashes;
       this.saveFileHashCache();
       stats.durationMs = Date.now() - startTime;
@@ -520,7 +530,7 @@ export class Indexer {
 
     if (pendingChunks.length === 0) {
       database.clearBranch(this.currentBranch);
-      database.addChunksToBranch(this.currentBranch, Array.from(currentChunkIds));
+      database.addChunksToBranchBatch(this.currentBranch, Array.from(currentChunkIds));
       store.save();
       invertedIndex.save();
       this.fileHashCache = currentFileHashes;
@@ -615,17 +625,15 @@ export class Indexer {
 
           store.addBatch(items);
 
-          for (let i = 0; i < batch.length; i++) {
-            const chunk = batch[i];
-            const embedding = result.embeddings[i];
-            
-            database.upsertEmbedding(
-              chunk.contentHash,
-              float32ArrayToBuffer(embedding),
-              chunk.text,
-              detectedProvider.modelInfo.model
-            );
-            
+          const embeddingBatchItems = batch.map((chunk, i) => ({
+            contentHash: chunk.contentHash,
+            embedding: float32ArrayToBuffer(result.embeddings[i]),
+            chunkText: chunk.text,
+            model: detectedProvider.modelInfo.model,
+          }));
+          database.upsertEmbeddingsBatch(embeddingBatchItems);
+
+          for (const chunk of batch) {
             invertedIndex.removeChunk(chunk.id);
             invertedIndex.addChunk(chunk.id, chunk.content);
           }
@@ -659,7 +667,7 @@ export class Indexer {
     });
 
     database.clearBranch(this.currentBranch);
-    database.addChunksToBranch(this.currentBranch, Array.from(currentChunkIds));
+    database.addChunksToBranchBatch(this.currentBranch, Array.from(currentChunkIds));
 
     store.save();
     invertedIndex.save();

src/native/index.ts

@@ -542,6 +542,18 @@ export class Database {
     this.inner.upsertEmbedding(contentHash, embedding, chunkText, model);
   }
 
+  upsertEmbeddingsBatch(
+    items: Array<{
+      contentHash: string;
+      embedding: Buffer;
+      chunkText: string;
+      model: string;
+    }>
+  ): void {
+    if (items.length === 0) return;
+    this.inner.upsertEmbeddingsBatch(items);
+  }
+
   getMissingEmbeddings(contentHashes: string[]): string[] {
     return this.inner.getMissingEmbeddings(contentHashes);
   }
@@ -550,6 +562,11 @@ export class Database {
     this.inner.upsertChunk(chunk);
   }
 
+  upsertChunksBatch(chunks: ChunkData[]): void {
+    if (chunks.length === 0) return;
+    this.inner.upsertChunksBatch(chunks);
+  }
+
   getChunk(chunkId: string): ChunkData | null {
     return this.inner.getChunk(chunkId) ?? null;
   }
@@ -566,6 +583,11 @@ export class Database {
     this.inner.addChunksToBranch(branch, chunkIds);
   }
 
+  addChunksToBranchBatch(branch: string, chunkIds: string[]): void {
+    if (chunkIds.length === 0) return;
+    this.inner.addChunksToBranchBatch(branch, chunkIds);
+  }
+
   clearBranch(branch: string): number {
     return this.inner.clearBranch(branch);
   }

tests/database.test.ts

@@ -293,4 +293,148 @@ describe("Database", () => {
       expect(stats.branchCount).toBe(1);
     });
   });
+
+  describe("batch operations", () => {
+    it("should upsert embeddings in batch", () => {
+      const items = [
+        {
+          contentHash: "batch_hash1",
+          embedding: Buffer.from(new Float32Array([1.0, 2.0]).buffer),
+          chunkText: "text1",
+          model: "test-model",
+        },
+        {
+          contentHash: "batch_hash2",
+          embedding: Buffer.from(new Float32Array([3.0, 4.0]).buffer),
+          chunkText: "text2",
+          model: "test-model",
+        },
+        {
+          contentHash: "batch_hash3",
+          embedding: Buffer.from(new Float32Array([5.0, 6.0]).buffer),
+          chunkText: "text3",
+          model: "test-model",
+        },
+      ];
+
+      db.upsertEmbeddingsBatch(items);
+
+      expect(db.embeddingExists("batch_hash1")).toBe(true);
+      expect(db.embeddingExists("batch_hash2")).toBe(true);
+      expect(db.embeddingExists("batch_hash3")).toBe(true);
+
+      const retrieved = db.getEmbedding("batch_hash2");
+      expect(retrieved).not.toBeNull();
+      const floats = new Float32Array(retrieved!.buffer, retrieved!.byteOffset, retrieved!.byteLength / 4);
+      expect(floats[0]).toBeCloseTo(3.0);
+      expect(floats[1]).toBeCloseTo(4.0);
+    });
+
+    it("should handle empty embeddings batch", () => {
+      db.upsertEmbeddingsBatch([]);
+      expect(db.getStats().embeddingCount).toBe(0);
+    });
+
+    it("should upsert chunks in batch", () => {
+      const chunks: ChunkData[] = [
+        {
+          chunkId: "batch_chunk1",
+          contentHash: "hash1",
+          filePath: "/file1.ts",
+          startLine: 1,
+          endLine: 10,
+          nodeType: "function",
+          name: "func1",
+          language: "typescript",
+        },
+        {
+          chunkId: "batch_chunk2",
+          contentHash: "hash2",
+          filePath: "/file2.ts",
+          startLine: 20,
+          endLine: 30,
+          nodeType: "class",
+          name: "MyClass",
+          language: "typescript",
+        },
+        {
+          chunkId: "batch_chunk3",
+          contentHash: "hash3",
+          filePath: "/file1.ts",
+          startLine: 50,
+          endLine: 60,
+          language: "typescript",
+        },
+      ];
+
+      db.upsertChunksBatch(chunks);
+
+      const chunk1 = db.getChunk("batch_chunk1");
+      expect(chunk1).not.toBeNull();
+      expect(chunk1!.filePath).toBe("/file1.ts");
+      expect(chunk1!.name).toBe("func1");
+
+      const chunk2 = db.getChunk("batch_chunk2");
+      expect(chunk2).not.toBeNull();
+      expect(chunk2!.nodeType).toBe("class");
+
+      const chunk3 = db.getChunk("batch_chunk3");
+      expect(chunk3).not.toBeNull();
+
+      const file1Chunks = db.getChunksByFile("/file1.ts");
+      expect(file1Chunks.length).toBe(2);
+    });
+
+    it("should handle empty chunks batch", () => {
+      db.upsertChunksBatch([]);
+      expect(db.getStats().chunkCount).toBe(0);
+    });
+
+    it("should add chunks to branch in batch", () => {
+      const chunks: ChunkData[] = [
+        { chunkId: "c1", contentHash: "h1", filePath: "/f.ts", startLine: 1, endLine: 5, language: "ts" },
+        { chunkId: "c2", contentHash: "h2", filePath: "/f.ts", startLine: 10, endLine: 15, language: "ts" },
+        { chunkId: "c3", contentHash: "h3", filePath: "/f.ts", startLine: 20, endLine: 25, language: "ts" },
+      ];
+      db.upsertChunksBatch(chunks);
+
+      db.addChunksToBranchBatch("feature-branch", ["c1", "c2", "c3"]);
+
+      const branchChunks = db.getBranchChunkIds("feature-branch");
+      expect(branchChunks.length).toBe(3);
+      expect(branchChunks).toContain("c1");
+      expect(branchChunks).toContain("c2");
+      expect(branchChunks).toContain("c3");
+    });
+
+    it("should handle empty branch batch", () => {
+      db.addChunksToBranchBatch("empty-branch", []);
+      expect(db.getBranchChunkIds("empty-branch").length).toBe(0);
+    });
+
+    it("should update existing chunks in batch", () => {
+      db.upsertChunk({
+        chunkId: "update_chunk",
+        contentHash: "old_hash",
+        filePath: "/old.ts",
+        startLine: 1,
+        endLine: 5,
+        language: "typescript",
+      });
+
+      db.upsertChunksBatch([{
+        chunkId: "update_chunk",
+        contentHash: "new_hash",
+        filePath: "/new.ts",
+        startLine: 10,
+        endLine: 20,
+        language: "typescript",
+      }]);
+
+      const chunk = db.getChunk("update_chunk");
+      expect(chunk!.contentHash).toBe("new_hash");
+      expect(chunk!.filePath).toBe("/new.ts");
+      expect(chunk!.startLine).toBe(10);
+    });
+  });
 });