Updated the benchmark

Author: ezimuel

BENCHMARK.md

@@ -10,7 +10,7 @@ PHPVector, measuring the four metrics that matter in production:
 | **P99 latency** | Worst-case latency for 99% of queries |
 | **Recall@k** | Fraction of true nearest neighbours returned |
 
-A **persistence section** is also included, measuring `persist()` and `load()` speed and file throughput.
+A **persistence section** is also included, measuring `save()` and `open()` speed and folder throughput.
 
 ---
 
@@ -101,7 +101,7 @@ php benchmark/benchmark.php [options]
 | Option | Default | Description |
 |--------|---------|-------------|
 | `--output=<file>` | stdout | Write Markdown report to this path |
-| `--no-persist` | off | Skip the persistence (save/load) phase |
+| `--no-save` | off | Skip the persistence (`save` / `open`) phase |
 | `--no-recall` | off | Skip recall computation (faster for large datasets) |
 | `--help`, `-h` | — | Print usage and exit |
 
@@ -194,11 +194,13 @@ A Recall@10 of 95% means HNSW returns 9–10 of the true 10 nearest neighbours
 on average. Values below ~80% suggest `efSearch` is too low for the dataset, or
 `M`/`efConstruction` need increasing.
 
-### Persistence (persist / load)
+### Persistence (save / open)
 
-Wall-clock time and throughput (MB/s) for `VectorDatabase::persist()` and
-`VectorDatabase::load()`. The `.phpv` binary format uses `pack/unpack` for
-float arrays, so throughput is typically limited by filesystem speed.
+Wall-clock time and throughput (MB/s) for `VectorDatabase::save()` and
+`VectorDatabase::open()`. `save()` waits for any outstanding async document
+writes, then flushes `hnsw.bin` and `bm25.bin`. `open()` reads only those two
+index files; individual document files (`docs/{n}.bin`) are loaded lazily after
+search, so open time is typically fast regardless of document count.
 
 ---
 
@@ -327,9 +329,10 @@ nodes for some queries.
 **Recall** — breakdown at k=1, k=5, and k=K. Recall@1 is always ≥ Recall@k
 because finding the single nearest neighbour is easier than finding the top-K.
 
-**Persistence** — file size on disk, time and throughput for `persist()` and
-`load()`. Useful for planning deployment workflows where the index is built once
-and served from disk on each restart.
+**Persistence** — total folder size on disk, time and throughput for `save()` and
+`open()`. Useful for planning deployment workflows where the index is built once
+and served from disk on each restart. `open()` time is typically much faster than
+`save()` because document files are not read eagerly.
 
 ---
 
@@ -342,7 +345,7 @@ and served from disk on each restart.
 | Recall@10 | ≥ 95% | < 85% — increase `efSearch` or `M` |
 | P99 / P50 ratio | < 3× | > 5× — graph may have poor connectivity |
 | Build throughput | Consistent with N·log(N) growth | Sudden drops may indicate GC pressure |
-| persist() throughput | > 50 MB/s | Lower suggests filesystem bottleneck |
+| save() throughput | > 50 MB/s | Lower suggests filesystem bottleneck |
 
 ### The recall/speed tradeoff
 

benchmark/Report.php

@@ -139,17 +139,17 @@ public static function generate(
                 $p = $result['persist'];
                 $lines[] = '### Persistence';
                 $lines[] = '';
-                $lines[] = '| Operation | File size | Time | Throughput |';
-                $lines[] = '|-----------|-----------|------|------------|';
-                $lines[] = sprintf('| `persist()` | %s | %s | %.1f MB/s |',
-                    self::fmtMb($p['file_size_mb']),
-                    self::fmtTime($p['persist_s']),
-                    $p['persist_mb_s'],
+                $lines[] = '| Operation | Folder size | Time | Throughput |';
+                $lines[] = '|-----------|-------------|------|------------|';
+                $lines[] = sprintf('| `save()` | %s | %s | %.1f MB/s |',
+                    self::fmtMb($p['folder_size_mb']),
+                    self::fmtTime($p['save_s']),
+                    $p['save_mb_s'],
                 );
-                $lines[] = sprintf('| `load()` | %s | %s | %.1f MB/s |',
-                    self::fmtMb($p['file_size_mb']),
-                    self::fmtTime($p['load_s']),
-                    $p['load_mb_s'],
+                $lines[] = sprintf('| `open()` | %s | %s | %.1f MB/s |',
+                    self::fmtMb($p['folder_size_mb']),
+                    self::fmtTime($p['open_s']),
+                    $p['open_mb_s'],
                 );
                 $lines[] = '';
             }

benchmark/benchmark.php

@@ -20,7 +20,7 @@
  *   --m=<n>                   HNSW M parameter                                 (default: 16)
  *   --seed=<n>                Random seed for reproducibility                  (default: 42)
  *   --output=<file>           Write Markdown report to file (default: stdout)
- *   --no-persist              Skip persistence benchmarks
+ *   --no-save                 Skip persistence benchmarks (save / open)
  *   --no-recall               Skip recall computation
  *   --help, -h                Show this help
  *
@@ -70,7 +70,7 @@
 $opts = getopt('h', [
     'scenarios:', 'k:', 'queries:', 'recall-samples:',
     'ef-search:', 'ef-construction:', 'm:', 'seed:',
-    'output:', 'no-persist', 'no-recall', 'help', 'h',
+    'output:', 'no-save', 'no-persist', 'no-recall', 'help', 'h',
 ]);
 
 if (isset($opts['help']) || isset($opts['h'])) {
@@ -100,7 +100,7 @@
 $m             = max(2, (int) ($opts['m'] ?? 16));
 $seed          = (int) ($opts['seed'] ?? 42);
 $outputFile    = $opts['output'] ?? null;
-$noPersist     = isset($opts['no-persist']);
+$noPersist     = isset($opts['no-save']) || isset($opts['no-persist']);
 $noRecall      = isset($opts['no-recall']);
 
 $hnswConfig = new HNSWConfig(
@@ -181,6 +181,33 @@ function computeRecall(
     return array_map(static fn(float $t): float => $t / $nRecall, $totals);
 }
 
+/**
+ * Recursively compute the total size of a directory in megabytes.
+ */
+function folderSizeMb(string $dir): float
+{
+    $bytes = 0;
+    $iter  = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS));
+    foreach ($iter as $file) {
+        $bytes += $file->getSize();
+    }
+    return $bytes / (1024 * 1024);
+}
+
+/**
+ * Recursively delete a directory and all its contents.
+ */
+function rrmdir(string $dir): void
+{
+    if (!is_dir($dir)) {
+        return;
+    }
+    foreach ((array) glob($dir . '/*') as $item) {
+        is_dir((string) $item) ? rrmdir((string) $item) : unlink((string) $item);
+    }
+    rmdir($dir);
+}
+
 /**
  * Run a complete benchmark for one scenario and return the result array.
  */
@@ -258,30 +285,41 @@ function runScenario(
     }
 
     // 6. Persistence ───────────────────────────────────────────────────────
+    //
+    // Build a fresh VectorDatabase with a temp folder path so document files
+    // are written async during insert.  save() flushes the HNSW graph and
+    // BM25 index (waiting for any outstanding async doc writes first).
+    // open() reads only hnsw.bin + bm25.bin — document files are lazy.
     $persist = null;
     if (!$noPersist) {
-        progress("  Benchmarking persist / load …\n");
+        progress("  Benchmarking save / open …\n");
+
+        $tmpDir = sys_get_temp_dir() . '/phpvbench_' . uniqid('', true);
+        mkdir($tmpDir, 0755, true);
 
-        $tmpFile = tempnam(sys_get_temp_dir(), 'phpvbench_') . '.phpv';
+        $dbSave = new VectorDatabase($hnswConfig, new BM25Config(), new SimpleTokenizer([]), $tmpDir);
+        for ($i = 0; $i < $n; $i++) {
+            $dbSave->addDocument(new Document(id: $i, vector: $dataVectors[$i]));
+        }
 
         $t0 = hrtime(true);
-        $db->persist($tmpFile);
-        $persistTime = (hrtime(true) - $t0) / 1e9;
+        $dbSave->save();
+        $saveTime = (hrtime(true) - $t0) / 1e9;
 
-        $fileSizeMb = filesize($tmpFile) / (1024 * 1024);
+        $folderSizeMb = folderSizeMb($tmpDir);
 
         $t0 = hrtime(true);
-        VectorDatabase::load($tmpFile, $hnswConfig);
-        $loadTime = (hrtime(true) - $t0) / 1e9;
+        VectorDatabase::open($tmpDir, $hnswConfig);
+        $openTime = (hrtime(true) - $t0) / 1e9;
 
-        unlink($tmpFile);
+        rrmdir($tmpDir);
 
         $persist = [
-            'file_size_mb'  => $fileSizeMb,
-            'persist_s'     => $persistTime,
-            'persist_mb_s'  => $persistTime > 0.0 ? $fileSizeMb / $persistTime : 0.0,
-            'load_s'        => $loadTime,
-            'load_mb_s'     => $loadTime > 0.0 ? $fileSizeMb / $loadTime : 0.0,
+            'folder_size_mb' => $folderSizeMb,
+            'save_s'         => $saveTime,
+            'save_mb_s'      => $saveTime > 0.0 ? $folderSizeMb / $saveTime : 0.0,
+            'open_s'         => $openTime,
+            'open_mb_s'      => $openTime > 0.0 ? $folderSizeMb / $openTime : 0.0,
         ];
     }
 

src/Persistence/IndexSerializer.php

@@ -66,7 +66,7 @@ public function writeHnsw(string $path, array $state): void
 
         $buf  = self::HNSW_MAGIC;
         $buf .= pack('C', self::VERSION);
-        $buf .= pack('NNNN', $dim, $nodeCount, $ep ?? self::NULL_ENTRY_POINT, (int) $state['maxLayer']);
+        $buf .= pack('NNNN', $dim, $nodeCount, $ep, (int) $state['maxLayer']);
 
         foreach ($nodes as $nodeId => $node) {
             $buf .= pack('NN', $nodeId, $node['maxLayer']);