Skip to content
This repository was archived by the owner on May 20, 2026. It is now read-only.

Commit b59229d

Browse files
author
Aashna Garg
committed
Add candidateModel and stickyOverride to automode.routerDecision telemetry
The automode.routerDecision event was only logging predictedLabel and confidence, making it impossible to distinguish what model the router recommended vs. what model was actually used after sticky-provider and vision overrides. New properties: - candidateModel: the router's top pick (candidate_models[0]) before any same-provider or vision fallback overrides are applied - stickyOverride: whether the router applied a sticky override (1/0) - conversationId: enables joining with response.success events - vscodeRequestId: enables joining with response.success events This allows analysts to detect when the router recommends model A but the client uses model B due to provider affinity or vision fallback.
1 parent bb0f664 commit b59229d

2 files changed

Lines changed: 12 additions & 4 deletions

File tree

src/platform/endpoint/node/automodeService.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ export class AutomodeService extends Disposable implements IAutomodeService {
368368
previous_model: entry?.endpoint?.model,
369369
turn_number: (entry?.turnCount ?? 0) + 1,
370370
};
371-
const result = await this._routerDecisionFetcher.getRouterDecision(prompt, token.session_token, token.available_models, undefined, contextSignals);
371+
const result = await this._routerDecisionFetcher.getRouterDecision(prompt, token.session_token, token.available_models, undefined, contextSignals, conversationId, chatRequest?.id);
372372

373373
if (!result.candidate_models.length) {
374374
return { lastRoutedPrompt: prompt, fallbackReason: 'emptyCandidateList' };
@@ -465,7 +465,7 @@ export class AutomodeService extends Disposable implements IAutomodeService {
465465
};
466466

467467
const result = await this._routerDecisionFetcher.getRouterDecision(
468-
prompt, token.session_token, token.available_models, undefined, contextSignals);
468+
prompt, token.session_token, token.available_models, undefined, contextSignals, conversationId);
469469

470470
if (!result.candidate_models.length) {
471471
this._logService.trace('[AutomodeService] resolveEndpointForSummarization: empty candidates');

src/platform/endpoint/node/routerDecisionFetcher.ts

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ export class RouterDecisionFetcher {
4848
) {
4949
}
5050

51-
async getRouterDecision(query: string, autoModeToken: string, availableModels: string[], stickyThreshold?: number, contextSignals?: RoutingContextSignals): Promise<RouterDecisionResponse> {
51+
async getRouterDecision(query: string, autoModeToken: string, availableModels: string[], stickyThreshold?: number, contextSignals?: RoutingContextSignals, conversationId?: string, vscodeRequestId?: string): Promise<RouterDecisionResponse> {
5252
const startTime = Date.now();
5353
const requestBody: Record<string, unknown> = { prompt: query, available_models: availableModels, ...contextSignals };
5454
if (stickyThreshold !== undefined) {
@@ -100,20 +100,28 @@ export class RouterDecisionFetcher {
100100
"automode.routerDecision" : {
101101
"owner": "lramos15",
102102
"comment": "Reports the routing decision made by the auto mode router API",
103+
"conversationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The conversation ID in which the routing decision was made" },
104+
"vscodeRequestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The VS Code chat request ID in which the routing decision was made" },
103105
"predictedLabel": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The predicted classification label (needs_reasoning or no_reasoning)" },
106+
"candidateModel": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The top candidate model recommended by the router before any overrides" },
104107
"confidence": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The confidence score of the routing decision" },
105108
"latencyMs": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true, "comment": "The latency of the router API call in milliseconds" },
106-
"e2eLatencyMs": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true, "comment": "The end-to-end latency of the router request in milliseconds, including network overhead" }
109+
"e2eLatencyMs": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true, "comment": "The end-to-end latency of the router request in milliseconds, including network overhead" },
110+
"stickyOverride": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Whether the router applied a sticky override (1) or not (0)" }
107111
}
108112
*/
109113
this._telemetryService.sendMSFTTelemetryEvent('automode.routerDecision',
110114
{
115+
conversationId: conversationId ?? '',
116+
vscodeRequestId: vscodeRequestId ?? '',
111117
predictedLabel: result.predicted_label,
118+
candidateModel: result.candidate_models[0] ?? '',
112119
},
113120
{
114121
confidence: result.confidence,
115122
latencyMs: result.latency_ms,
116123
e2eLatencyMs: e2eLatencyMs,
124+
stickyOverride: result.sticky_override ? 1 : 0,
117125
}
118126
);
119127
return result;

0 commit comments

Comments
 (0)