fix: allow eval diff to read legacy summaries

Author: Helweg

src/eval/cli.ts

@@ -336,8 +336,12 @@ export async function handleEvalCommand(args: string[], cwd: string): Promise<nu
     if (!parsed.againstPath.endsWith(".json")) {
       throw new Error("eval diff --against must point to a summary JSON file");
     }
-    const currentSummary = loadSummary(path.resolve(parsed.projectRoot, currentPath));
-    const baselineSummary = loadSummary(path.resolve(parsed.projectRoot, parsed.againstPath));
+    const currentSummary = loadSummary(path.resolve(parsed.projectRoot, currentPath), {
+      allowLegacyDiversityMetrics: true,
+    });
+    const baselineSummary = loadSummary(path.resolve(parsed.projectRoot, parsed.againstPath), {
+      allowLegacyDiversityMetrics: true,
+    });
     const comparison = compareSummaries(
       currentSummary,
       baselineSummary,

src/eval/reports.ts

@@ -9,22 +9,43 @@ import type {
   SweepAggregateReport,
 } from "./types.js";
 
+interface LoadSummaryOptions {
+  allowLegacyDiversityMetrics?: boolean;
+}
+
 function assertFiniteNumber(value: unknown, path: string): number {
   if (typeof value !== "number" || Number.isNaN(value) || !Number.isFinite(value)) {
     throw new Error(`${path} must be a finite number`);
   }
   return value;
 }
 
-function validateSummary(summary: EvalSummary, summaryPath: string): EvalSummary {
+function validateSummary(
+  summary: EvalSummary,
+  summaryPath: string,
+  options?: LoadSummaryOptions
+): EvalSummary {
   assertFiniteNumber(summary.metrics.hitAt1, `${summaryPath}.metrics.hitAt1`);
   assertFiniteNumber(summary.metrics.hitAt3, `${summaryPath}.metrics.hitAt3`);
   assertFiniteNumber(summary.metrics.hitAt5, `${summaryPath}.metrics.hitAt5`);
   assertFiniteNumber(summary.metrics.hitAt10, `${summaryPath}.metrics.hitAt10`);
   assertFiniteNumber(summary.metrics.mrrAt10, `${summaryPath}.metrics.mrrAt10`);
   assertFiniteNumber(summary.metrics.ndcgAt10, `${summaryPath}.metrics.ndcgAt10`);
-  assertFiniteNumber(summary.metrics.distinctTop3Ratio, `${summaryPath}.metrics.distinctTop3Ratio`);
-  assertFiniteNumber(summary.metrics.rawDistinctTop3Ratio, `${summaryPath}.metrics.rawDistinctTop3Ratio`);
+
+  const metrics = summary.metrics as EvalSummary["metrics"] & {
+    distinctTop3Ratio?: number;
+    rawDistinctTop3Ratio?: number;
+  };
+
+  if (metrics.distinctTop3Ratio === undefined && options?.allowLegacyDiversityMetrics) {
+    metrics.distinctTop3Ratio = 0;
+  }
+  if (metrics.rawDistinctTop3Ratio === undefined && options?.allowLegacyDiversityMetrics) {
+    metrics.rawDistinctTop3Ratio = 0;
+  }
+
+  assertFiniteNumber(metrics.distinctTop3Ratio, `${summaryPath}.metrics.distinctTop3Ratio`);
+  assertFiniteNumber(metrics.rawDistinctTop3Ratio, `${summaryPath}.metrics.rawDistinctTop3Ratio`);
   assertFiniteNumber(summary.metrics.latencyMs.p50, `${summaryPath}.metrics.latencyMs.p50`);
   assertFiniteNumber(summary.metrics.latencyMs.p95, `${summaryPath}.metrics.latencyMs.p95`);
   assertFiniteNumber(summary.metrics.latencyMs.p99, `${summaryPath}.metrics.latencyMs.p99`);
@@ -51,9 +72,9 @@ function signed(value: number, digits = 4): string {
   return value > 0 ? `+${formatted}` : formatted;
 }
 
-export function loadSummary(summaryPath: string): EvalSummary {
+export function loadSummary(summaryPath: string, options?: LoadSummaryOptions): EvalSummary {
   const raw = readFileSync(summaryPath, "utf-8");
-  return validateSummary(JSON.parse(raw) as EvalSummary, summaryPath);
+  return validateSummary(JSON.parse(raw) as EvalSummary, summaryPath, options);
 }
 
 export function createRunDirectory(outputRoot: string, timestampOverride?: string): string {

tests/eval-cli.test.ts

@@ -108,6 +108,8 @@ describe("eval cli", () => {
         hitAt10: 1,
         mrrAt10: 1,
         ndcgAt10: 1,
+        distinctTop3Ratio: 1,
+        rawDistinctTop3Ratio: 1,
         latencyMs: { p50: 1, p95: 2, p99: 3 },
         tokenEstimate: { queryTokens: 10, embeddingTokensUsed: 20 },
         embedding: { callCount: 1, estimatedCostUsd: 0, costPer1MTokensUsd: 0 },
@@ -130,4 +132,52 @@ describe("eval cli", () => {
 
     expect(exitCode).toBe(0);
   });
+
+  it("allows eval diff to read legacy summaries missing diversity metrics", async () => {
+    const currentSummaryPath = path.join(tempDir, "current.json");
+    const baselineSummaryPath = path.join(tempDir, "baseline.json");
+
+    const legacySummary = {
+      generatedAt: new Date().toISOString(),
+      projectRoot: tempDir,
+      datasetPath: "benchmarks/golden/small.json",
+      datasetName: "small",
+      datasetVersion: "1.0.0",
+      queryCount: 1,
+      topK: 10,
+      searchConfig: {
+        fusionStrategy: "rrf",
+        hybridWeight: 0.4,
+        rrfK: 60,
+        rerankTopN: 20,
+      },
+      metrics: {
+        hitAt1: 1,
+        hitAt3: 1,
+        hitAt5: 1,
+        hitAt10: 1,
+        mrrAt10: 1,
+        ndcgAt10: 1,
+        latencyMs: { p50: 1, p95: 2, p99: 3 },
+        tokenEstimate: { queryTokens: 10, embeddingTokensUsed: 20 },
+        embedding: { callCount: 1, estimatedCostUsd: 0, costPer1MTokensUsd: 0 },
+        failureBuckets: {
+          "wrong-file": 0,
+          "wrong-symbol": 0,
+          "docs-tests-outranking-source": 0,
+          "no-relevant-hit-top-k": 0,
+        },
+      },
+    };
+
+    writeFileSync(currentSummaryPath, JSON.stringify(legacySummary, null, 2), "utf-8");
+    writeFileSync(baselineSummaryPath, JSON.stringify(legacySummary, null, 2), "utf-8");
+
+    const exitCode = await handleEvalCommand(
+      ["diff", "--current", "current.json", "--against", "baseline.json"],
+      tempDir
+    );
+
+    expect(exitCode).toBe(0);
+  });
 });