fix(token-search): renumber inverted index after doc removal

FuseIndex.removeAll renumbers surviving records contiguously, but the
inverted index only deleted the old docIdx entries — postings and
docTerms keys for docs past the removal point kept stale indices.
Later removes or token scoring could then operate on wrong docs.

Adds removeAndShiftInvertedIndex which removes the given docIdx
entries, then renumbers surviving postings.docIdx and docTerms keys
in a single pass using a binary-search shift. Wired into Fuse.remove
and Fuse.removeAt; removeFromInvertedIndex is kept for direct use
in tests.

Covers single remove with shift, non-contiguous batch remove, add
after shift, and full-collection remove.

Author: krisk

src/core/index.ts

@@ -12,7 +12,7 @@ import { createAnalyzer } from '../search/token/analyzer'
 import {
   buildInvertedIndex,
   addToInvertedIndex,
-  removeFromInvertedIndex
+  removeAndShiftInvertedIndex
 } from '../search/token/InvertedIndex'
 import type { InvertedIndexData } from '../search/token/InvertedIndex'
 import type {
@@ -160,9 +160,7 @@ export default class Fuse<T> {
 
     if (indicesToRemove.length) {
       if (this._invertedIndex) {
-        for (const idx of indicesToRemove) {
-          removeFromInvertedIndex(this._invertedIndex, idx)
-        }
+        removeAndShiftInvertedIndex(this._invertedIndex, indicesToRemove)
       }
 
       // Filter docs in a single pass instead of reverse-splicing
@@ -176,7 +174,7 @@ export default class Fuse<T> {
 
   removeAt(idx: number): T {
     if (this._invertedIndex) {
-      removeFromInvertedIndex(this._invertedIndex, idx)
+      removeAndShiftInvertedIndex(this._invertedIndex, [idx])
     }
     const doc = this._docs.splice(idx, 1)[0]
     this._myIndex.removeAt(idx)

src/search/token/InvertedIndex.ts

@@ -186,3 +186,48 @@ export function removeFromInvertedIndex(
   index.fieldCount -= docFields.size
   index.docTerms.delete(docIdx)
 }
+
+// Removes the given docIdx entries and renumbers remaining postings/docTerms
+// so that they stay in sync with FuseIndex's contiguous renumbering on remove.
+export function removeAndShiftInvertedIndex(
+  index: InvertedIndexData,
+  removedIndices: number[]
+): void {
+  if (removedIndices.length === 0) return
+
+  // De-dup and sort so the shift computation is O(log k) per lookup.
+  const sorted = Array.from(new Set(removedIndices)).sort((a, b) => a - b)
+
+  for (const idx of sorted) {
+    removeFromInvertedIndex(index, idx)
+  }
+
+  // For any surviving oldIdx, its new idx is oldIdx minus the number of
+  // removed indices strictly less than oldIdx.
+  const shift = (oldIdx: number): number => {
+    let lo = 0
+    let hi = sorted.length
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1
+      if (sorted[mid] < oldIdx) lo = mid + 1
+      else hi = mid
+    }
+    return oldIdx - lo
+  }
+
+  const firstRemoved = sorted[0]
+
+  for (const postings of index.terms.values()) {
+    for (const p of postings) {
+      if (p.docIdx > firstRemoved) {
+        p.docIdx = shift(p.docIdx)
+      }
+    }
+  }
+
+  const shiftedDocTerms = new Map<number, Set<string>>()
+  for (const [oldKey, terms] of index.docTerms) {
+    shiftedDocTerms.set(oldKey > firstRemoved ? shift(oldKey) : oldKey, terms)
+  }
+  index.docTerms = shiftedDocTerms
+}

test/internals.test.ts

@@ -2,7 +2,8 @@ import norm from '../src/tools/fieldNorm'
 import {
   buildInvertedIndex,
   addToInvertedIndex,
-  removeFromInvertedIndex
+  removeFromInvertedIndex,
+  removeAndShiftInvertedIndex
 } from '../src/search/token/InvertedIndex'
 import { createAnalyzer } from '../src/search/token/analyzer'
 import type { IndexRecord } from '../src/types'
@@ -135,4 +136,78 @@ describe('InvertedIndex', () => {
     removeFromInvertedIndex(index, 0)
     expect(index.docTerms.has(0)).toBe(false)
   })
+
+  test('removeAndShiftInvertedIndex renumbers surviving postings', () => {
+    const records = makeRecords()
+    const index = buildInvertedIndex(records, 2, analyzer)
+
+    // Remove doc 0 — docs 1, 2 must shift down to 0, 1
+    removeAndShiftInvertedIndex(index, [0])
+
+    expect(index.docTerms.has(0)).toBe(true)
+    expect(index.docTerms.has(1)).toBe(true)
+    expect(index.docTerms.has(2)).toBe(false)
+    expect(index.docTerms.get(0)).toEqual(new Set(['hello', 'there']))
+    expect(index.docTerms.get(1)).toEqual(new Set(['goodbye', 'cruel', 'world']))
+
+    const helloPostings = index.terms.get('hello')!
+    expect(helloPostings.length).toBe(1)
+    expect(helloPostings[0].docIdx).toBe(0)
+
+    const worldPostings = index.terms.get('world')!
+    expect(worldPostings.length).toBe(1)
+    expect(worldPostings[0].docIdx).toBe(1)
+  })
+
+  test('batch remove + shift handles non-contiguous indices', () => {
+    const records: IndexRecord[] = [
+      { i: 0, $: { 0: { v: 'alpha', n: 1 } } },
+      { i: 1, $: { 0: { v: 'beta', n: 1 } } },
+      { i: 2, $: { 0: { v: 'gamma', n: 1 } } },
+      { i: 3, $: { 0: { v: 'delta', n: 1 } } },
+      { i: 4, $: { 0: { v: 'epsilon', n: 1 } } }
+    ]
+    const index = buildInvertedIndex(records, 1, analyzer)
+
+    // Remove docs 1 and 3 — remaining (0, 2, 4) become (0, 1, 2)
+    removeAndShiftInvertedIndex(index, [1, 3])
+
+    expect(index.docTerms.get(0)).toEqual(new Set(['alpha']))
+    expect(index.docTerms.get(1)).toEqual(new Set(['gamma']))
+    expect(index.docTerms.get(2)).toEqual(new Set(['epsilon']))
+    expect(index.docTerms.has(3)).toBe(false)
+    expect(index.docTerms.has(4)).toBe(false)
+
+    expect(index.terms.get('gamma')![0].docIdx).toBe(1)
+    expect(index.terms.get('epsilon')![0].docIdx).toBe(2)
+  })
+
+  test('removing every doc leaves an empty index', () => {
+    const records = makeRecords()
+    const index = buildInvertedIndex(records, 2, analyzer)
+
+    removeAndShiftInvertedIndex(index, [0, 1, 2])
+
+    expect(index.fieldCount).toBe(0)
+    expect(index.docTerms.size).toBe(0)
+    expect(index.terms.size).toBe(0)
+    expect(index.df.size).toBe(0)
+  })
+
+  test('subsequent add after shift uses the new contiguous index', () => {
+    const records = makeRecords()
+    const index = buildInvertedIndex(records, 2, analyzer)
+
+    removeAndShiftInvertedIndex(index, [0])
+
+    // After shift: 2 surviving docs at indices 0, 1. A new add gets index 2.
+    const newRecord: IndexRecord = {
+      i: 2,
+      $: { 0: { v: 'fresh doc', n: 1 } }
+    }
+    addToInvertedIndex(index, newRecord, 2, analyzer)
+
+    expect(index.docTerms.get(2)).toEqual(new Set(['fresh', 'doc']))
+    expect(index.terms.get('fresh')![0].docIdx).toBe(2)
+  })
 })