fix(reference): replace non-deterministic archive verification with per-file hashes

Summary:
The reference warmup model verification was re-compressing extracted model
files into a .tgz and comparing the archive's SHA-256 and size against
values baked into embeddings.json. This is fundamentally non-deterministic
because gzip output varies across Node versions, tar package versions, OS
zlib implementations, and CPU architectures. Users on different machines
than the build server would get checksum mismatches and be blocked from
using reference tools.

Replaced the archive-based verification with per-file SHA-256 hashing:
- Added optional `fileHashes: Record<string, string>` to the model
  metadata type, mapping each model file's relative path to its SHA-256.
- Removed `createDeterministicModelArchive` and `readInstalledModelMetadata`
  from assets.ts (the runtime verification path).
- Added `validateInstalledModelFiles` that checks each file individually.
- Updated `getReferenceCacheStatus` and `installPinnedModelFiles` to use
  per-file verification instead of re-archiving.
- Updated the build-time tools (model.ts, pinned-model.mjs, build-model.mjs)
  to compute and include fileHashes in the model metadata.
- Old corpus formats without fileHashes gracefully degrade to file-existence
  checks only, which is strictly better than the broken archive comparison.

Reviewed By: cabanier

Differential Revision: D105088189

fbshipit-source-id: 83a245f3938108ec56f6083884e54c359c3312a5

Author: felixtrz

packages/reference-assets/pinned-model.mjs

@@ -54,11 +54,13 @@ export const REQUIRED_MODEL_FILES = Object.freeze(
 export function buildReferenceEmbeddingModelMetadata(
   archiveSha256,
   archiveSize,
+  fileHashes,
 ) {
   return {
     ...DEFAULT_REFERENCE_MODEL_SETTINGS,
     archiveSha256,
     archiveSize,
+    ...(fileHashes ? { fileHashes } : {}),
   };
 }
 
@@ -86,6 +88,16 @@ export async function sha256File(filePath) {
   });
 }
 
+export async function computeModelFileHashes(modelDir) {
+  const hashes = {};
+  for (const file of REFERENCE_MODEL_FILE_SOURCES) {
+    hashes[file.relativePath] = await sha256File(
+      path.join(modelDir, file.relativePath),
+    );
+  }
+  return hashes;
+}
+
 async function writeResponseToFile(response, destination, sourceUrl) {
   if (!response.body) {
     throw new Error(`No response body received from ${sourceUrl}`);

packages/reference-assets/scripts/build-model.mjs

@@ -10,6 +10,7 @@ import fsp from 'node:fs/promises';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import {
+  computeModelFileHashes,
   createDeterministicModelArchive,
   downloadPinnedModelFile,
   REFERENCE_MODEL_FILE_SOURCES,
@@ -70,6 +71,7 @@ async function main() {
 
     const archiveStat = await fsp.stat(archivePath);
     const archiveSha256 = await sha256File(archivePath);
+    const fileHashes = await computeModelFileHashes(sourceDir);
     const manifest = {
       schemaVersion: 1,
       referenceVersion: packageJson.version,
@@ -80,6 +82,7 @@ async function main() {
         size: archiveStat.size,
         format: 'transformers-js',
         requiredFiles: REQUIRED_MODEL_FILES,
+        fileHashes,
       },
     };
 

packages/reference-assets/test/model.test.ts

@@ -21,10 +21,15 @@ import {
 let tempDir: string;
 let sharedRoot: string;
 
-async function createModelArchive(): Promise<{ sha256: string; size: number }> {
+async function createModelArchive(): Promise<{
+  sha256: string;
+  size: number;
+  fileHashes: Record<string, string>;
+}> {
   const sourceRoot = path.join(tempDir, 'model-source');
   const archivePath = path.join(tempDir, 'model.tgz');
   const archiveRoot = path.join(sourceRoot, 'model');
+  const fileHashes: Record<string, string> = {};
 
   await rm(sourceRoot, { recursive: true, force: true });
   await mkdir(archiveRoot, { recursive: true });
@@ -40,6 +45,7 @@ async function createModelArchive(): Promise<{ sha256: string; size: number }> {
     const destination = path.join(archiveRoot, relativePath);
     await mkdir(path.dirname(destination), { recursive: true });
     await writeFile(destination, body);
+    fileHashes[relativePath] = createHash('sha256').update(body).digest('hex');
   }
 
   await tar.c(
@@ -58,6 +64,7 @@ async function createModelArchive(): Promise<{ sha256: string; size: number }> {
   return {
     sha256: createHash('sha256').update(buffer).digest('hex'),
     size: buffer.length,
+    fileHashes,
   };
 }
 const ONNX_BUFFER = Buffer.from('fake-onnx');
@@ -124,6 +131,7 @@ describe('reference model installer', () => {
       pooling: 'mean',
       normalize: true,
     });
+    expect(installed.metadata.fileHashes).toEqual(expectedArchive.fileHashes);
     expect(formatReferenceEmbeddingModel(installed.metadata)).toBe(
       `sha256:${expectedArchive.sha256}`,
     );

packages/reference-assets/tools/model.ts

@@ -35,6 +35,7 @@ export interface ReferenceEmbeddingModelMetadata {
   format: 'transformers-js';
   archiveSha256: string;
   archiveSize: number;
+  fileHashes?: Record<string, string>;
   dtype: ReferenceEmbeddingModelDType;
   pooling: 'mean';
   normalize: true;
@@ -88,11 +89,13 @@ const REQUIRED_MODEL_FILES = Object.freeze(
 function buildReferenceEmbeddingModelMetadata(
   archiveSha256: string,
   archiveSize: number,
+  fileHashes?: Record<string, string>,
 ): ReferenceEmbeddingModelMetadata {
   return {
     ...DEFAULT_REFERENCE_MODEL_SETTINGS,
     archiveSha256,
     archiveSize,
+    ...(fileHashes ? { fileHashes } : {}),
   };
 }
 
@@ -246,27 +249,44 @@ async function createDeterministicModelArchive(
   );
 }
 
-async function readInstalledModelMetadata(
+async function computeModelFileHashes(
   modelDir: string,
-  stagingRoot = getReferenceModelStagingRoot(),
-): Promise<ReferenceEmbeddingModelMetadata> {
-  const archivePath = path.join(
-    stagingRoot,
-    'verification',
-    `model-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.tgz`,
-  );
-  await mkdir(path.dirname(archivePath), { recursive: true });
-
-  try {
-    await createDeterministicModelArchive(modelDir, archivePath);
-    const archiveStat = await stat(archivePath);
-    return buildReferenceEmbeddingModelMetadata(
-      await sha256File(archivePath),
-      archiveStat.size,
+): Promise<Record<string, string>> {
+  const hashes: Record<string, string> = {};
+  for (const file of REFERENCE_MODEL_FILE_SOURCES) {
+    hashes[file.relativePath] = await sha256File(
+      path.join(modelDir, file.relativePath),
     );
-  } finally {
-    await rm(archivePath, { force: true }).catch(() => {});
   }
+  return hashes;
+}
+
+async function validateInstalledModelFiles(
+  modelDir: string,
+  expectedFileHashes: Record<string, string> | undefined,
+): Promise<boolean> {
+  if (!hasReferenceEmbeddingModelFiles(modelDir)) {
+    return false;
+  }
+
+  if (!expectedFileHashes) {
+    return true;
+  }
+
+  for (const [relativePath, expectedSha] of Object.entries(
+    expectedFileHashes,
+  )) {
+    try {
+      const actualSha = await sha256File(path.join(modelDir, relativePath));
+      if (actualSha !== expectedSha) {
+        return false;
+      }
+    } catch {
+      return false;
+    }
+  }
+
+  return true;
 }
 
 export interface InstalledReferenceEmbeddingModel {
@@ -323,16 +343,10 @@ async function installReferenceModelFiles(
     metadata.archiveSha256,
   );
   const finalDir = path.join(finalRoot, 'model');
-  if (hasReferenceEmbeddingModelFiles(finalDir)) {
-    const installedMetadata = await readInstalledModelMetadata(finalDir);
-    if (
-      installedMetadata.archiveSha256 === metadata.archiveSha256 &&
-      installedMetadata.archiveSize === metadata.archiveSize
-    ) {
-      return finalDir;
-    }
-    await rm(finalRoot, { recursive: true, force: true });
+  if (await validateInstalledModelFiles(finalDir, metadata.fileHashes)) {
+    return finalDir;
   }
+  await rm(finalRoot, { recursive: true, force: true });
 
   if (!hasReferenceEmbeddingModelFiles(extractedDir)) {
     throw new Error(
@@ -349,13 +363,8 @@ async function installReferenceModelFiles(
     await rename(tempFinalRoot, finalRoot);
   } catch (error) {
     await rm(tempFinalRoot, { recursive: true, force: true });
-    if (!hasReferenceEmbeddingModelFiles(finalDir)) {
-      throw error;
-    }
-    const installedMetadata = await readInstalledModelMetadata(finalDir);
     if (
-      installedMetadata.archiveSha256 !== metadata.archiveSha256 ||
-      installedMetadata.archiveSize !== metadata.archiveSize
+      !(await validateInstalledModelFiles(finalDir, metadata.fileHashes))
     ) {
       throw error;
     }
@@ -382,11 +391,14 @@ export async function installReferenceEmbeddingModel(): Promise<InstalledReferen
       await downloadPinnedModelFile(file.sourceUrl, destination);
     }
 
+    const fileHashes = await computeModelFileHashes(extractedDir);
+
     await createDeterministicModelArchive(extractedDir, archivePath);
     const archiveStat = await stat(archivePath);
     const metadata = buildReferenceEmbeddingModelMetadata(
       await sha256File(archivePath),
       archiveStat.size,
+      fileHashes,
     );
     const modelDir = await installReferenceModelFiles(
       metadata,

packages/reference/src/assets.ts

@@ -18,7 +18,6 @@ import os from 'os';
 import path from 'path';
 import * as tar from 'tar';
 import {
-  buildReferenceEmbeddingModelMetadata,
   hasReferenceEmbeddingModelFiles,
   REFERENCE_MODEL_FILE_SOURCES,
   REFERENCE_MODEL_ONNX_URL,
@@ -459,43 +458,30 @@ async function validateModelDir(modelDir: string): Promise<boolean> {
   return hasReferenceEmbeddingModelFiles(modelDir);
 }
 
-async function createDeterministicModelArchive(
-  sourceDir: string,
-  archivePath: string,
-): Promise<void> {
-  await tar.c(
-    {
-      cwd: path.dirname(sourceDir),
-      file: archivePath,
-      gzip: true,
-      portable: true,
-      noPax: true,
-      mtime: new Date(0),
-    },
-    [path.basename(sourceDir)],
-  );
-}
-
-async function readInstalledModelMetadata(
+async function validateInstalledModelFiles(
   modelDir: string,
-): Promise<ReferenceEmbeddingModelMetadata> {
-  const archivePath = path.join(
-    getStagingRoot(),
-    'verification',
-    `model-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.tgz`,
-  );
-  await mkdir(path.dirname(archivePath), { recursive: true });
+  expectedFileHashes: Record<string, string> | undefined,
+): Promise<boolean> {
+  if (!(await validateModelDir(modelDir))) {
+    return false;
+  }
 
-  try {
-    await createDeterministicModelArchive(modelDir, archivePath);
-    const archiveStat = await stat(archivePath);
-    return buildReferenceEmbeddingModelMetadata(
-      await sha256File(archivePath),
-      archiveStat.size,
-    );
-  } finally {
-    await rm(archivePath, { force: true }).catch(() => {});
+  if (!expectedFileHashes) {
+    return true;
   }
+
+  for (const [relativePath, expectedSha] of Object.entries(expectedFileHashes)) {
+    try {
+      const actualSha = await sha256File(path.join(modelDir, relativePath));
+      if (actualSha !== expectedSha) {
+        return false;
+      }
+    } catch {
+      return false;
+    }
+  }
+
+  return true;
 }
 
 async function readCurrentModelMetadata(
@@ -622,23 +608,13 @@ export async function getReferenceCacheStatus(): Promise<ReferenceCacheStatus> {
         );
       }
 
-      if (!(await validateModelDir(state.modelDir))) {
-        return buildFailureStatus(
-          packageVersion,
-          state,
-          'Reference model cache is incomplete or corrupted. Run "iwsdk reference warmup" again.',
-        );
-      }
-
-      const installedModel = await readInstalledModelMetadata(state.modelDir);
       if (
-        installedModel.archiveSha256 !== model.archiveSha256 ||
-        installedModel.archiveSize !== model.archiveSize
+        !(await validateInstalledModelFiles(state.modelDir, model.fileHashes))
       ) {
         return buildFailureStatus(
           packageVersion,
           state,
-          `Reference model cache metadata ${installedModel.archiveSha256}/${installedModel.archiveSize} does not match the warmed corpus metadata ${model.archiveSha256}/${model.archiveSize}. Run "iwsdk reference warmup" again to refresh the pinned model files.`,
+          'Reference model cache is incomplete or corrupted. Run "iwsdk reference warmup" again.',
         );
       }
 
@@ -1087,19 +1063,12 @@ async function installPinnedModelFiles(
     metadata.archiveSha256,
   );
   const finalDir = path.join(finalRoot, 'model');
-  if (await validateModelDir(finalDir)) {
-    const installedModel = await readInstalledModelMetadata(finalDir);
-    if (
-      installedModel.archiveSha256 === metadata.archiveSha256 &&
-      installedModel.archiveSize === metadata.archiveSize
-    ) {
-      return finalDir;
-    }
-    await rm(finalRoot, { recursive: true, force: true });
+  if (await validateInstalledModelFiles(finalDir, metadata.fileHashes)) {
+    return finalDir;
   }
+  await rm(finalRoot, { recursive: true, force: true });
 
   const extractedDir = path.join(stagingRoot, 'model-extract', 'model');
-  const archivePath = path.join(stagingRoot, 'model.tgz');
   await rm(path.dirname(extractedDir), { recursive: true, force: true });
   await mkdir(path.join(extractedDir, 'onnx'), { recursive: true });
 
@@ -1115,18 +1084,19 @@ async function installPinnedModelFiles(
     );
   }
 
-  await createDeterministicModelArchive(extractedDir, archivePath);
-  const archiveStat = await stat(archivePath);
-  const archiveSha256 = await sha256File(archivePath);
-  if (archiveStat.size !== metadata.archiveSize) {
-    throw new Error(
-      `Pinned reference model archive size ${archiveStat.size} does not match the warmed corpus metadata ${metadata.archiveSize}. Run "iwsdk reference warmup" again to refresh the pinned model files.`,
-    );
-  }
-  if (archiveSha256 !== metadata.archiveSha256) {
-    throw new Error(
-      `Pinned reference model archive sha ${archiveSha256} does not match the warmed corpus metadata ${metadata.archiveSha256}. Run "iwsdk reference warmup" again to refresh the pinned model files.`,
-    );
+  if (metadata.fileHashes) {
+    for (const [relativePath, expectedSha] of Object.entries(
+      metadata.fileHashes,
+    )) {
+      const actualSha = await sha256File(
+        path.join(extractedDir, relativePath),
+      );
+      if (actualSha !== expectedSha) {
+        throw new Error(
+          `Pinned reference model file ${relativePath} has unexpected content after download. Run "iwsdk reference warmup" again.`,
+        );
+      }
+    }
   }
 
   await mkdir(path.dirname(finalRoot), { recursive: true });
@@ -1138,13 +1108,8 @@ async function installPinnedModelFiles(
     await rename(tempFinalRoot, finalRoot);
   } catch (error) {
     await rm(tempFinalRoot, { recursive: true, force: true });
-    if (!(await validateModelDir(finalDir))) {
-      throw error;
-    }
-    const installedModel = await readInstalledModelMetadata(finalDir);
     if (
-      installedModel.archiveSha256 !== metadata.archiveSha256 ||
-      installedModel.archiveSize !== metadata.archiveSize
+      !(await validateInstalledModelFiles(finalDir, metadata.fileHashes))
     ) {
       throw error;
     }