fix: reject legacy eval baselines missing diversity metrics

Author: Helweg

src/eval/reports.ts

@@ -9,6 +9,31 @@ import type {
   SweepAggregateReport,
 } from "./types.js";
 
+function assertFiniteNumber(value: unknown, path: string): number {
+  if (typeof value !== "number" || Number.isNaN(value) || !Number.isFinite(value)) {
+    throw new Error(`${path} must be a finite number`);
+  }
+  return value;
+}
+
+function validateSummary(summary: EvalSummary, summaryPath: string): EvalSummary {
+  assertFiniteNumber(summary.metrics.hitAt1, `${summaryPath}.metrics.hitAt1`);
+  assertFiniteNumber(summary.metrics.hitAt3, `${summaryPath}.metrics.hitAt3`);
+  assertFiniteNumber(summary.metrics.hitAt5, `${summaryPath}.metrics.hitAt5`);
+  assertFiniteNumber(summary.metrics.hitAt10, `${summaryPath}.metrics.hitAt10`);
+  assertFiniteNumber(summary.metrics.mrrAt10, `${summaryPath}.metrics.mrrAt10`);
+  assertFiniteNumber(summary.metrics.ndcgAt10, `${summaryPath}.metrics.ndcgAt10`);
+  assertFiniteNumber(summary.metrics.distinctTop3Ratio, `${summaryPath}.metrics.distinctTop3Ratio`);
+  assertFiniteNumber(summary.metrics.rawDistinctTop3Ratio, `${summaryPath}.metrics.rawDistinctTop3Ratio`);
+  assertFiniteNumber(summary.metrics.latencyMs.p50, `${summaryPath}.metrics.latencyMs.p50`);
+  assertFiniteNumber(summary.metrics.latencyMs.p95, `${summaryPath}.metrics.latencyMs.p95`);
+  assertFiniteNumber(summary.metrics.latencyMs.p99, `${summaryPath}.metrics.latencyMs.p99`);
+  assertFiniteNumber(summary.metrics.embedding.callCount, `${summaryPath}.metrics.embedding.callCount`);
+  assertFiniteNumber(summary.metrics.embedding.estimatedCostUsd, `${summaryPath}.metrics.embedding.estimatedCostUsd`);
+
+  return summary;
+}
+
 function formatPct(value: number): string {
   return `${(value * 100).toFixed(2)}%`;
 }
@@ -28,7 +53,7 @@ function signed(value: number, digits = 4): string {
 
 export function loadSummary(summaryPath: string): EvalSummary {
   const raw = readFileSync(summaryPath, "utf-8");
-  return JSON.parse(raw) as EvalSummary;
+  return validateSummary(JSON.parse(raw) as EvalSummary, summaryPath);
 }
 
 export function createRunDirectory(outputRoot: string, timestampOverride?: string): string {

tests/eval-runner.test.ts

@@ -161,6 +161,94 @@ describe("eval runner", () => {
     expect(readFileSync(path.join(compareRun.outputDir, "compare.json"), "utf-8")).toContain("\"deltas\"");
   });
 
+  it("fails fast when baseline summary is missing required diversity metrics", async () => {
+    const baselineRun = await runEvaluation({
+      projectRoot: tempDir,
+      datasetPath: "benchmarks/golden/small.json",
+      outputRoot: "benchmarks/results",
+      ciMode: false,
+      reindex: false,
+    });
+
+    const legacyBaseline = {
+      ...baselineRun.summary,
+      metrics: {
+        ...baselineRun.summary.metrics,
+      },
+    } as Record<string, unknown>;
+
+    delete (legacyBaseline.metrics as Record<string, unknown>).distinctTop3Ratio;
+    delete (legacyBaseline.metrics as Record<string, unknown>).rawDistinctTop3Ratio;
+
+    const baselinePath = path.join(tempDir, "benchmarks", "baselines", "legacy-baseline-summary.json");
+    writeFileSync(baselinePath, JSON.stringify(legacyBaseline, null, 2), "utf-8");
+
+    await expect(
+      runEvaluation({
+        projectRoot: tempDir,
+        datasetPath: "benchmarks/golden/small.json",
+        outputRoot: "benchmarks/results",
+        againstPath: "benchmarks/baselines/legacy-baseline-summary.json",
+        ciMode: false,
+        reindex: false,
+      })
+    ).rejects.toThrow(/metrics\.distinctTop3Ratio must be a finite number/);
+  });
+
+  it("fails ci mode when budget baseline summary is missing required diversity metrics", async () => {
+    const baselineRun = await runEvaluation({
+      projectRoot: tempDir,
+      datasetPath: "benchmarks/golden/small.json",
+      outputRoot: "benchmarks/results",
+      ciMode: false,
+      reindex: false,
+    });
+
+    const legacyBaseline = {
+      ...baselineRun.summary,
+      metrics: {
+        ...baselineRun.summary.metrics,
+      },
+    } as Record<string, unknown>;
+
+    delete (legacyBaseline.metrics as Record<string, unknown>).distinctTop3Ratio;
+    delete (legacyBaseline.metrics as Record<string, unknown>).rawDistinctTop3Ratio;
+
+    writeFileSync(
+      path.join(tempDir, "benchmarks", "baselines", "legacy-baseline-summary.json"),
+      JSON.stringify(legacyBaseline, null, 2),
+      "utf-8"
+    );
+
+    writeFileSync(
+      path.join(tempDir, "benchmarks", "budgets", "legacy-check.json"),
+      JSON.stringify(
+        {
+          name: "legacy-check",
+          baselinePath: "benchmarks/baselines/legacy-baseline-summary.json",
+          failOnMissingBaseline: true,
+          thresholds: {
+            rawDistinctTop3RatioMaxDrop: 0.1,
+          },
+        },
+        null,
+        2
+      ),
+      "utf-8"
+    );
+
+    await expect(
+      runEvaluation({
+        projectRoot: tempDir,
+        datasetPath: "benchmarks/golden/small.json",
+        outputRoot: "benchmarks/results",
+        ciMode: true,
+        budgetPath: "benchmarks/budgets/legacy-check.json",
+        reindex: false,
+      })
+    ).rejects.toThrow(/metrics\.distinctTop3Ratio must be a finite number/);
+  });
+
   it("fails ci gate when thresholds regress beyond tolerance", async () => {
     const baselineRun = await runEvaluation({
       projectRoot: tempDir,