Skip to content

Commit 5929af6

Browse files
committed
fix: skip consecutive spaces in fieldNorm word counting
The space-counting loop counted every space character, so "hello world" (4 spaces) would report 5 tokens instead of 2. Changed to track space→non-space transitions so consecutive spaces are treated as a single word separator, matching the behavior of the original regex.
1 parent 5800036 commit 5929af6

12 files changed

Lines changed: 71 additions & 17 deletions

dist/fuse.basic.cjs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -221,10 +221,18 @@ function norm(weight = 1, mantissa = 3) {
221221
const m = Math.pow(10, mantissa);
222222
return {
223223
get(value) {
224-
// Count words by counting spaces — avoids allocating a regex match array
224+
// Count words by counting space transitions — avoids allocating a regex match array
225225
let numTokens = 1;
226+
let inSpace = false;
226227
for (let i = 0; i < value.length; i++) {
227-
if (value.charCodeAt(i) === 32) numTokens++;
228+
if (value.charCodeAt(i) === 32) {
229+
if (!inSpace) {
230+
numTokens++;
231+
inSpace = true;
232+
}
233+
} else {
234+
inSpace = false;
235+
}
228236
}
229237
if (cache.has(numTokens)) {
230238
return cache.get(numTokens);

dist/fuse.basic.min.cjs

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

dist/fuse.basic.min.mjs

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

dist/fuse.basic.mjs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,10 +219,18 @@ function norm(weight = 1, mantissa = 3) {
219219
const m = Math.pow(10, mantissa);
220220
return {
221221
get(value) {
222-
// Count words by counting spaces — avoids allocating a regex match array
222+
// Count words by counting space transitions — avoids allocating a regex match array
223223
let numTokens = 1;
224+
let inSpace = false;
224225
for (let i = 0; i < value.length; i++) {
225-
if (value.charCodeAt(i) === 32) numTokens++;
226+
if (value.charCodeAt(i) === 32) {
227+
if (!inSpace) {
228+
numTokens++;
229+
inSpace = true;
230+
}
231+
} else {
232+
inSpace = false;
233+
}
226234
}
227235
if (cache.has(numTokens)) {
228236
return cache.get(numTokens);

dist/fuse.cjs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -218,10 +218,18 @@ function norm(weight = 1, mantissa = 3) {
218218
const m = Math.pow(10, mantissa);
219219
return {
220220
get(value) {
221-
// Count words by counting spaces — avoids allocating a regex match array
221+
// Count words by counting space transitions — avoids allocating a regex match array
222222
let numTokens = 1;
223+
let inSpace = false;
223224
for (let i = 0; i < value.length; i++) {
224-
if (value.charCodeAt(i) === 32) numTokens++;
225+
if (value.charCodeAt(i) === 32) {
226+
if (!inSpace) {
227+
numTokens++;
228+
inSpace = true;
229+
}
230+
} else {
231+
inSpace = false;
232+
}
225233
}
226234
if (cache.has(numTokens)) {
227235
return cache.get(numTokens);

dist/fuse.min.cjs

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

dist/fuse.min.mjs

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

dist/fuse.mjs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -216,10 +216,18 @@ function norm(weight = 1, mantissa = 3) {
216216
const m = Math.pow(10, mantissa);
217217
return {
218218
get(value) {
219-
// Count words by counting spaces — avoids allocating a regex match array
219+
// Count words by counting space transitions — avoids allocating a regex match array
220220
let numTokens = 1;
221+
let inSpace = false;
221222
for (let i = 0; i < value.length; i++) {
222-
if (value.charCodeAt(i) === 32) numTokens++;
223+
if (value.charCodeAt(i) === 32) {
224+
if (!inSpace) {
225+
numTokens++;
226+
inSpace = true;
227+
}
228+
} else {
229+
inSpace = false;
230+
}
223231
}
224232
if (cache.has(numTokens)) {
225233
return cache.get(numTokens);

dist/fuse.worker.min.mjs

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

dist/fuse.worker.mjs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -216,10 +216,18 @@ function norm(weight = 1, mantissa = 3) {
216216
const m = Math.pow(10, mantissa);
217217
return {
218218
get(value) {
219-
// Count words by counting spaces — avoids allocating a regex match array
219+
// Count words by counting space transitions — avoids allocating a regex match array
220220
let numTokens = 1;
221+
let inSpace = false;
221222
for (let i = 0; i < value.length; i++) {
222-
if (value.charCodeAt(i) === 32) numTokens++;
223+
if (value.charCodeAt(i) === 32) {
224+
if (!inSpace) {
225+
numTokens++;
226+
inSpace = true;
227+
}
228+
} else {
229+
inSpace = false;
230+
}
223231
}
224232
if (cache.has(numTokens)) {
225233
return cache.get(numTokens);

0 commit comments

Comments
 (0)