Skip to content

Commit 7821847

Browse files
yannbfyannbf
authored
Merge pull request #34537 from storybookjs/sidnioulz/agentic-telemetry-ws2
Core: Agentic observability for vitest and ghost stories
2 parents 5b6aa67 + e81a5db commit 7821847

29 files changed

+1035
-211
lines changed

‎code/addons/a11y/src/preview.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ export const afterEach: AfterEach<any> = async ({
2020
}) => {
2121
const a11yParameter: A11yParameters | undefined = parameters.a11y;
2222
const a11yGlobals = globals.a11y;
23+
// we do not run a11y checks as part of ghost stories runs
2324
const isGhostStories = !!globals.ghostStories;
2425

2526
const shouldRunEnvironmentIndependent =

‎code/addons/vitest/src/vitest-plugin/agent-telemetry-reporter.test.ts‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
import { beforeEach, describe, expect, it, vi } from 'vitest';
2+
3+
import { AgentTelemetryReporter } from './agent-telemetry-reporter.ts';
4+
5+
vi.mock('storybook/internal/telemetry', () => ({
6+
telemetry: vi.fn(),
7+
isExampleStoryId: vi.fn(
8+
(id: string) =>
9+
id.startsWith('example-button--') ||
10+
id.startsWith('example-header--') ||
11+
id.startsWith('example-page--')
12+
),
13+
}));
14+
15+
const { telemetry } = await import('storybook/internal/telemetry');
16+
17+
function createMockTestCase({
18+
storyId,
19+
status,
20+
reports = [],
21+
errors = [],
22+
}: {
23+
storyId?: string;
24+
status: 'passed' | 'failed' | 'pending';
25+
reports?: Array<{ type: string; result?: Record<string, unknown> }>;
26+
errors?: Array<{ message: string; stack?: string }>;
27+
}) {
28+
return {
29+
meta: () => ({ storyId, reports }),
30+
result: () => ({
31+
state: status,
32+
errors: status === 'failed' ? errors : [],
33+
}),
34+
};
35+
}
36+
37+
function createMockTestModules(testCounts: { passed: number; failed: number }) {
38+
const tests: Array<{ result: () => { state: string } }> = [];
39+
for (let i = 0; i < testCounts.passed; i++) {
40+
tests.push({ result: () => ({ state: 'passed' }) });
41+
}
42+
for (let i = 0; i < testCounts.failed; i++) {
43+
tests.push({ result: () => ({ state: 'failed' }) });
44+
}
45+
return [
46+
{
47+
children: {
48+
allTests: function* (filter?: string) {
49+
for (const t of tests) {
50+
if (!filter || t.result().state === filter) {
51+
yield t;
52+
}
53+
}
54+
},
55+
},
56+
errors: () => [],
57+
},
58+
];
59+
}
60+
61+
describe('AgentTelemetryReporter', () => {
62+
let reporter: AgentTelemetryReporter;
63+
64+
beforeEach(() => {
65+
vi.clearAllMocks();
66+
reporter = new AgentTelemetryReporter({
67+
configDir: '.storybook',
68+
agent: { name: 'claude' },
69+
});
70+
});
71+
72+
describe('onTestCaseResult', () => {
73+
it('should collect story test results', () => {
74+
const testCase = createMockTestCase({
75+
storyId: 'my-story--primary',
76+
status: 'passed',
77+
});
78+
reporter.onTestCaseResult(testCase as any);
79+
});
80+
81+
it('should skip tests without storyId', () => {
82+
const testCase = createMockTestCase({
83+
storyId: undefined,
84+
status: 'passed',
85+
});
86+
reporter.onTestCaseResult(testCase as any);
87+
});
88+
89+
it('should skip example story IDs', () => {
90+
const testCase = createMockTestCase({
91+
storyId: 'example-button--primary',
92+
status: 'passed',
93+
});
94+
reporter.onTestCaseResult(testCase as any);
95+
});
96+
});
97+
98+
describe('onTestRunEnd', () => {
99+
it('should send telemetry with analysis of collected results', async () => {
100+
reporter.onInit({ config: { watch: false } } as any);
101+
102+
reporter.onTestCaseResult(createMockTestCase({ storyId: 's1', status: 'passed' }) as any);
103+
reporter.onTestCaseResult(
104+
createMockTestCase({
105+
storyId: 's2',
106+
status: 'failed',
107+
errors: [{ message: 'Error: Module not found: foo' }],
108+
}) as any
109+
);
110+
reporter.onTestCaseResult(
111+
createMockTestCase({
112+
storyId: 's3',
113+
status: 'passed',
114+
reports: [{ type: 'render-analysis', result: { emptyRender: true } }],
115+
}) as any
116+
);
117+
118+
await reporter.onTestRunEnd(createMockTestModules({ passed: 2, failed: 1 }) as any, []);
119+
120+
expect(telemetry).toHaveBeenCalledWith(
121+
'ai-setup-self-healing-scoring',
122+
expect.objectContaining({
123+
agent: { name: 'claude' },
124+
analysis: expect.objectContaining({
125+
total: 3,
126+
passed: 2,
127+
passedButEmptyRender: 1,
128+
successRate: 0.67,
129+
successRateWithoutEmptyRender: 0.33,
130+
uniqueErrorCount: 1,
131+
}),
132+
unhandledErrorCount: 0,
133+
watch: false,
134+
}),
135+
{ configDir: '.storybook', stripMetadata: true }
136+
);
137+
});
138+
139+
it('should filter out example stories from analysis', async () => {
140+
reporter.onInit({ config: { watch: false } } as any);
141+
142+
reporter.onTestCaseResult(
143+
createMockTestCase({ storyId: 'my-story--primary', status: 'passed' }) as any
144+
);
145+
reporter.onTestCaseResult(
146+
createMockTestCase({ storyId: 'example-button--primary', status: 'passed' }) as any
147+
);
148+
149+
await reporter.onTestRunEnd(createMockTestModules({ passed: 2, failed: 0 }) as any, []);
150+
151+
expect(telemetry).toHaveBeenCalledWith(
152+
'ai-setup-self-healing-scoring',
153+
expect.objectContaining({
154+
analysis: expect.objectContaining({
155+
total: 1,
156+
passed: 1,
157+
}),
158+
}),
159+
expect.anything()
160+
);
161+
});
162+
163+
it('should count unhandled errors', async () => {
164+
reporter.onInit({ config: { watch: false } } as any);
165+
166+
await reporter.onTestRunEnd(
167+
createMockTestModules({ passed: 0, failed: 0 }) as any,
168+
[{ message: 'unhandled' }, { message: 'another' }] as any
169+
);
170+
171+
expect(telemetry).toHaveBeenCalledWith(
172+
'ai-setup-self-healing-scoring',
173+
expect.objectContaining({
174+
unhandledErrorCount: 2,
175+
}),
176+
expect.anything()
177+
);
178+
});
179+
180+
it('should reset collected results after each run', async () => {
181+
reporter.onInit({ config: { watch: false } } as any);
182+
183+
reporter.onTestCaseResult(createMockTestCase({ storyId: 's1', status: 'passed' }) as any);
184+
await reporter.onTestRunEnd(createMockTestModules({ passed: 1, failed: 0 }) as any, []);
185+
186+
reporter.onTestCaseResult(
187+
createMockTestCase({
188+
storyId: 's2',
189+
status: 'failed',
190+
errors: [{ message: 'err' }],
191+
}) as any
192+
);
193+
await reporter.onTestRunEnd(createMockTestModules({ passed: 0, failed: 1 }) as any, []);
194+
195+
const secondCall = vi.mocked(telemetry).mock.calls[1];
196+
expect(secondCall[1]).toEqual(
197+
expect.objectContaining({
198+
analysis: expect.objectContaining({
199+
total: 1,
200+
passed: 0,
201+
}),
202+
})
203+
);
204+
});
205+
});
206+
});

‎code/addons/vitest/src/vitest-plugin/agent-telemetry-reporter.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import type { SerializedError } from 'vitest';
2+
import type { TestCase, TestModule, Vitest } from 'vitest/node';
3+
import type { Reporter } from 'vitest/reporters';
4+
5+
import type { TaskMeta } from '@vitest/runner';
6+
import type { Report } from 'storybook/preview-api';
7+
import { analyzeTestResults, toStoryTestResult } from 'storybook/internal/core-server';
8+
import type { StoryTestResult } from 'storybook/internal/core-server';
9+
import { isExampleStoryId, telemetry } from 'storybook/internal/telemetry';
10+
import type { AgentInfo } from 'storybook/internal/telemetry';
11+
12+
interface AgentTelemetryReporterOptions {
13+
configDir: string;
14+
agent: AgentInfo;
15+
}
16+
17+
export class AgentTelemetryReporter implements Reporter {
18+
private ctx!: Vitest;
19+
20+
private testResults: StoryTestResult[] = [];
21+
22+
private startTime = Date.now();
23+
24+
private configDir: string;
25+
26+
private agent: AgentInfo;
27+
28+
constructor(options: AgentTelemetryReporterOptions) {
29+
this.configDir = options.configDir;
30+
this.agent = options.agent;
31+
}
32+
33+
onInit(ctx: Vitest) {
34+
this.ctx = ctx;
35+
}
36+
37+
onTestRunStart() {
38+
this.startTime = Date.now();
39+
}
40+
41+
onTestCaseResult(testCase: TestCase) {
42+
const { storyId, reports } = testCase.meta() as TaskMeta &
43+
Partial<{ storyId: string; reports: Report[] }>;
44+
45+
if (!storyId || isExampleStoryId(storyId)) {
46+
return;
47+
}
48+
49+
const testResult = testCase.result();
50+
const result = toStoryTestResult({
51+
storyId,
52+
statusRaw: testResult.state,
53+
reports,
54+
errors: testResult.errors,
55+
});
56+
57+
if (result) {
58+
this.testResults.push(result);
59+
}
60+
}
61+
62+
async onTestRunEnd(
63+
testModules: readonly TestModule[],
64+
unhandledErrors: readonly SerializedError[]
65+
) {
66+
const analysis = analyzeTestResults(this.testResults);
67+
const duration = Date.now() - this.startTime;
68+
69+
const testModulesErrors = testModules.flatMap((t) => t.errors());
70+
const unhandledErrorCount = unhandledErrors.length + testModulesErrors.length;
71+
72+
// Fire and forget — same pattern as the existing test-run telemetry
73+
telemetry(
74+
'ai-setup-self-healing-scoring',
75+
{
76+
agent: this.agent,
77+
analysis,
78+
unhandledErrorCount,
79+
duration,
80+
watch: this.ctx.config.watch,
81+
},
82+
{ configDir: this.configDir, stripMetadata: true }
83+
);
84+
85+
// Reset for next run (watch mode)
86+
this.testResults = [];
87+
}
88+
}

‎code/addons/vitest/src/vitest-plugin/index.ts‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,14 @@ import {
2020
} from 'storybook/internal/core-server';
2121
import { componentTransform, readConfig, vitestTransform } from 'storybook/internal/csf-tools';
2222
import { MainFileMissingError } from 'storybook/internal/server-errors';
23-
import { setTelemetryEnabled, telemetry } from 'storybook/internal/telemetry';
24-
import { oneWayHash } from 'storybook/internal/telemetry';
23+
import {
24+
detectAgent,
25+
isTelemetryModuleEnabled,
26+
isWithinInitialSession,
27+
oneWayHash,
28+
telemetry,
29+
setTelemetryEnabled,
30+
} from 'storybook/internal/telemetry';
2531
import type { Presets } from 'storybook/internal/types';
2632

2733
import { match } from 'micromatch';
@@ -36,6 +42,7 @@ import type { PluginOption } from 'vite';
3642
import { withoutVitePlugins } from '../../../../builders/builder-vite/src/utils/without-vite-plugins.ts';
3743
import type { InternalOptions, UserOptions } from './types.ts';
3844
import { requiresProjectAnnotations } from './utils.ts';
45+
import { AgentTelemetryReporter } from './agent-telemetry-reporter.ts';
3946

4047
const WORKING_DIR = process.cwd();
4148

@@ -241,6 +248,8 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
241248
plugins.push(mdxStubPlugin);
242249
}
243250

251+
let withinAgenticSetupSession = false;
252+
244253
const storybookTestPlugin: Plugin = {
245254
name: 'vite-plugin-storybook-test',
246255
async transformIndexHtml(html) {
@@ -385,6 +394,15 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
385394
globals.ghostStories = {
386395
enabled: true,
387396
};
397+
globals.renderAnalysis = {
398+
enabled: true,
399+
};
400+
}
401+
402+
if (withinAgenticSetupSession) {
403+
globals.renderAnalysis = {
404+
enabled: true,
405+
};
388406
}
389407

390408
return globals;
@@ -441,7 +459,7 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
441459
// return the new config, it will be deep-merged by vite
442460
return config;
443461
},
444-
configureVitest(context) {
462+
async configureVitest(context) {
445463
context.vitest.config.coverage.exclude.push('storybook-static');
446464

447465
// NOTE: we start telemetry immediately but do not wait on it. Typically it should complete
@@ -455,6 +473,21 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
455473
},
456474
{ configDir: finalOptions.configDir }
457475
);
476+
477+
if (isTelemetryModuleEnabled()) {
478+
// When an agent is running vitest via CLI, inject a reporter that sends
479+
// detailed test result telemetry (pass/fail, error analysis, empty renders)
480+
const agent = detectAgent();
481+
withinAgenticSetupSession = !!agent && (await isWithinInitialSession('ai-setup'));
482+
if (agent && withinAgenticSetupSession) {
483+
context.vitest.config.reporters.push(
484+
new AgentTelemetryReporter({
485+
configDir: finalOptions.configDir,
486+
agent,
487+
})
488+
);
489+
}
490+
}
458491
},
459492
async configureServer(server) {
460493
if (staticDirs) {

‎code/core/src/core-server/index.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,9 @@ export {
3737
} from './stores/test-provider.ts';
3838

3939
export { getComponentCandidates } from './utils/ghost-stories/get-candidates.ts';
40-
export { runGhostStories } from './utils/ghost-stories/run-story-tests.ts';
40+
export { runStoryTests } from './utils/ghost-stories/run-story-tests.ts';
4141
export { getServerPort } from './utils/server-address.ts';
42+
43+
export { analyzeTestResults } from '../shared/utils/analyze-test-results.ts';
44+
export type { StoryTestResult } from '../shared/utils/test-result-types.ts';
45+
export { toStoryTestResult } from '../shared/utils/to-story-test-result.ts';

0 commit comments

Comments
 (0)
⚡