opencode-codebase-index/src/eval/budget.ts at 926c828208fc1f904078bbb1d773cc48f1283815 · Helweg/opencode-codebase-index · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
import type { EvalBudget, EvalComparison, EvalGateResult, EvalSummary } from "./types.js";

export function evaluateBudgetGate(
  budget: EvalBudget,
  summary: EvalSummary,
  comparison?: EvalComparison
): EvalGateResult {
  const BASELINE_P95_EPSILON_MS = 0.001;
  const violations: EvalGateResult["violations"] = [];

  const { thresholds } = budget;

  if (thresholds.minHitAt5 !== undefined && summary.metrics.hitAt5 < thresholds.minHitAt5) {
    violations.push({
      metric: "minHitAt5",
      message: `Hit@5 ${summary.metrics.hitAt5.toFixed(4)} is below minimum ${thresholds.minHitAt5.toFixed(4)}`,
    });
  }

  if (thresholds.minMrrAt10 !== undefined && summary.metrics.mrrAt10 < thresholds.minMrrAt10) {
    violations.push({
      metric: "minMrrAt10",
      message: `MRR@10 ${summary.metrics.mrrAt10.toFixed(4)} is below minimum ${thresholds.minMrrAt10.toFixed(4)}`,
    });
  }

  if (
    thresholds.minRawDistinctTop3Ratio !== undefined &&
    summary.metrics.rawDistinctTop3Ratio < thresholds.minRawDistinctTop3Ratio
  ) {
    violations.push({
      metric: "minRawDistinctTop3Ratio",
      message: `Raw Distinct Top@3 ${summary.metrics.rawDistinctTop3Ratio.toFixed(4)} is below minimum ${thresholds.minRawDistinctTop3Ratio.toFixed(4)}`,
    });
  }

  if (comparison) {
    if (
      thresholds.hitAt5MaxDrop !== undefined &&
      comparison.deltas.hitAt5.absolute < -thresholds.hitAt5MaxDrop
    ) {
      violations.push({
        metric: "hitAt5MaxDrop",
        message: `Hit@5 drop ${comparison.deltas.hitAt5.absolute.toFixed(4)} exceeds allowed -${thresholds.hitAt5MaxDrop.toFixed(4)}`,
      });
    }

    if (
      thresholds.mrrAt10MaxDrop !== undefined &&
      comparison.deltas.mrrAt10.absolute < -thresholds.mrrAt10MaxDrop
    ) {
      violations.push({
        metric: "mrrAt10MaxDrop",
        message: `MRR@10 drop ${comparison.deltas.mrrAt10.absolute.toFixed(4)} exceeds allowed -${thresholds.mrrAt10MaxDrop.toFixed(4)}`,
      });
    }

    if (
      thresholds.rawDistinctTop3RatioMaxDrop !== undefined &&
      comparison.deltas.rawDistinctTop3Ratio.absolute < -thresholds.rawDistinctTop3RatioMaxDrop
    ) {
      violations.push({
        metric: "rawDistinctTop3RatioMaxDrop",
        message: `Raw Distinct Top@3 drop ${comparison.deltas.rawDistinctTop3Ratio.absolute.toFixed(4)} exceeds allowed -${thresholds.rawDistinctTop3RatioMaxDrop.toFixed(4)}`,
      });
    }

    if (thresholds.p95LatencyMaxMultiplier !== undefined) {
      const baselineP95 = comparison.deltas.latencyP95Ms.baseline;
      if (baselineP95 > BASELINE_P95_EPSILON_MS) {
        const allowed = baselineP95 * thresholds.p95LatencyMaxMultiplier;
        if (summary.metrics.latencyMs.p95 > allowed) {
          violations.push({
            metric: "p95LatencyMaxMultiplier",
            message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds allowed ${allowed.toFixed(3)}ms (${thresholds.p95LatencyMaxMultiplier.toFixed(2)}x baseline)`,
          });
        }
      }
    }
  }

  if (
    thresholds.p95LatencyMaxAbsoluteMs !== undefined &&
    summary.metrics.latencyMs.p95 > thresholds.p95LatencyMaxAbsoluteMs
  ) {
    violations.push({
      metric: "p95LatencyMaxAbsoluteMs",
      message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds absolute maximum ${thresholds.p95LatencyMaxAbsoluteMs.toFixed(3)}ms`,
    });
  }

  return {
    passed: violations.length === 0,
    budgetName: budget.name,
    violations,
  };
}