Core: Agentic observability for vitest and ghost stories

code/addons/a11y/src/preview.tsx

@@ -20,6 +20,7 @@ export const afterEach: AfterEach<any> = async ({
 }) => {
   const a11yParameter: A11yParameters | undefined = parameters.a11y;
   const a11yGlobals = globals.a11y;
+  // we do not run a11y checks as part of ghost stories runs
   const isGhostStories = !!globals.ghostStories;
 
   const shouldRunEnvironmentIndependent =

code/addons/vitest/src/vitest-plugin/agent-telemetry-reporter.test.ts

@@ -0,0 +1,206 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { AgentTelemetryReporter } from './agent-telemetry-reporter.ts';
+
+vi.mock('storybook/internal/telemetry', () => ({
+  telemetry: vi.fn(),
+  isExampleStoryId: vi.fn(
+    (id: string) =>
+      id.startsWith('example-button--') ||
+      id.startsWith('example-header--') ||
+      id.startsWith('example-page--')
+  ),
+}));
+
+const { telemetry } = await import('storybook/internal/telemetry');
+
+function createMockTestCase({
+  storyId,
+  status,
+  reports = [],
+  errors = [],
+}: {
+  storyId?: string;
+  status: 'passed' | 'failed' | 'pending';
+  reports?: Array<{ type: string; result?: Record<string, unknown> }>;
+  errors?: Array<{ message: string; stack?: string }>;
+}) {
+  return {
+    meta: () => ({ storyId, reports }),
+    result: () => ({
+      state: status,
+      errors: status === 'failed' ? errors : [],
+    }),
+  };
+}
+
+function createMockTestModules(testCounts: { passed: number; failed: number }) {
+  const tests: Array<{ result: () => { state: string } }> = [];
+  for (let i = 0; i < testCounts.passed; i++) {
+    tests.push({ result: () => ({ state: 'passed' }) });
+  }
+  for (let i = 0; i < testCounts.failed; i++) {
+    tests.push({ result: () => ({ state: 'failed' }) });
+  }
+  return [
+    {
+      children: {
+        allTests: function* (filter?: string) {
+          for (const t of tests) {
+            if (!filter || t.result().state === filter) {
+              yield t;
+            }
+          }
+        },
+      },
+      errors: () => [],
+    },
+  ];
+}
+
+describe('AgentTelemetryReporter', () => {
+  let reporter: AgentTelemetryReporter;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    reporter = new AgentTelemetryReporter({
+      configDir: '.storybook',
+      agent: { name: 'claude' },
+    });
+  });
+
+  describe('onTestCaseResult', () => {
+    it('should collect story test results', () => {
+      const testCase = createMockTestCase({
+        storyId: 'my-story--primary',
+        status: 'passed',
+      });
+      reporter.onTestCaseResult(testCase as any);
+    });
+
+    it('should skip tests without storyId', () => {
+      const testCase = createMockTestCase({
+        storyId: undefined,
+        status: 'passed',
+      });
+      reporter.onTestCaseResult(testCase as any);
+    });
+
+    it('should skip example story IDs', () => {
+      const testCase = createMockTestCase({
+        storyId: 'example-button--primary',
+        status: 'passed',
+      });
+      reporter.onTestCaseResult(testCase as any);
+    });
+  });
+
+  describe('onTestRunEnd', () => {
+    it('should send telemetry with analysis of collected results', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      reporter.onTestCaseResult(createMockTestCase({ storyId: 's1', status: 'passed' }) as any);
+      reporter.onTestCaseResult(
+        createMockTestCase({
+          storyId: 's2',
+          status: 'failed',
+          errors: [{ message: 'Error: Module not found: foo' }],
+        }) as any
+      );
+      reporter.onTestCaseResult(
+        createMockTestCase({
+          storyId: 's3',
+          status: 'passed',
+          reports: [{ type: 'render-analysis', result: { emptyRender: true } }],
+        }) as any
+      );
+
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 2, failed: 1 }) as any, []);
+
+      expect(telemetry).toHaveBeenCalledWith(
+        'ai-setup-self-healing-scoring',
+        expect.objectContaining({
+          agent: { name: 'claude' },
+          analysis: expect.objectContaining({
+            total: 3,
+            passed: 2,
+            passedButEmptyRender: 1,
+            successRate: 0.67,
+            successRateWithoutEmptyRender: 0.33,
+            uniqueErrorCount: 1,
+          }),
+          unhandledErrorCount: 0,
+          watch: false,
+        }),
+        { configDir: '.storybook', stripMetadata: true }
+      );
+    });
+
+    it('should filter out example stories from analysis', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      reporter.onTestCaseResult(
+        createMockTestCase({ storyId: 'my-story--primary', status: 'passed' }) as any
+      );
+      reporter.onTestCaseResult(
+        createMockTestCase({ storyId: 'example-button--primary', status: 'passed' }) as any
+      );
+
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 2, failed: 0 }) as any, []);
+
+      expect(telemetry).toHaveBeenCalledWith(
+        'ai-setup-self-healing-scoring',
+        expect.objectContaining({
+          analysis: expect.objectContaining({
+            total: 1,
+            passed: 1,
+          }),
+        }),
+        expect.anything()
+      );
+    });
+
+    it('should count unhandled errors', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      await reporter.onTestRunEnd(
+        createMockTestModules({ passed: 0, failed: 0 }) as any,
+        [{ message: 'unhandled' }, { message: 'another' }] as any
+      );
+
+      expect(telemetry).toHaveBeenCalledWith(
+        'ai-setup-self-healing-scoring',
+        expect.objectContaining({
+          unhandledErrorCount: 2,
+        }),
+        expect.anything()
+      );
+    });
+
+    it('should reset collected results after each run', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      reporter.onTestCaseResult(createMockTestCase({ storyId: 's1', status: 'passed' }) as any);
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 1, failed: 0 }) as any, []);
+
+      reporter.onTestCaseResult(
+        createMockTestCase({
+          storyId: 's2',
+          status: 'failed',
+          errors: [{ message: 'err' }],
+        }) as any
+      );
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 0, failed: 1 }) as any, []);
+
+      const secondCall = vi.mocked(telemetry).mock.calls[1];
+      expect(secondCall[1]).toEqual(
+        expect.objectContaining({
+          analysis: expect.objectContaining({
+            total: 1,
+            passed: 0,
+          }),
+        })
+      );
+    });
+  });
+});

code/addons/vitest/src/vitest-plugin/agent-telemetry-reporter.ts

@@ -0,0 +1,88 @@
+import type { SerializedError } from 'vitest';
+import type { TestCase, TestModule, Vitest } from 'vitest/node';
+import type { Reporter } from 'vitest/reporters';
+
+import type { TaskMeta } from '@vitest/runner';
+import type { Report } from 'storybook/preview-api';
+import { analyzeTestResults, toStoryTestResult } from 'storybook/internal/core-server';
+import type { StoryTestResult } from 'storybook/internal/core-server';
+import { isExampleStoryId, telemetry } from 'storybook/internal/telemetry';
+import type { AgentInfo } from 'storybook/internal/telemetry';
+
+interface AgentTelemetryReporterOptions {
+  configDir: string;
+  agent: AgentInfo;
+}
+
+export class AgentTelemetryReporter implements Reporter {
+  private ctx!: Vitest;
+
+  private testResults: StoryTestResult[] = [];
+
+  private startTime = Date.now();
+
+  private configDir: string;
+
+  private agent: AgentInfo;
+
+  constructor(options: AgentTelemetryReporterOptions) {
+    this.configDir = options.configDir;
+    this.agent = options.agent;
+  }
+
+  onInit(ctx: Vitest) {
+    this.ctx = ctx;
+  }
+
+  onTestRunStart() {
+    this.startTime = Date.now();
+  }
+
+  onTestCaseResult(testCase: TestCase) {
+    const { storyId, reports } = testCase.meta() as TaskMeta &
+      Partial<{ storyId: string; reports: Report[] }>;
+
+    if (!storyId || isExampleStoryId(storyId)) {
+      return;
+    }
+
+    const testResult = testCase.result();
+    const result = toStoryTestResult({
+      storyId,
+      statusRaw: testResult.state,
+      reports,
+      errors: testResult.errors,
+    });
+
+    if (result) {
+      this.testResults.push(result);
+    }
+  }
+
+  async onTestRunEnd(
+    testModules: readonly TestModule[],
+    unhandledErrors: readonly SerializedError[]
+  ) {
+    const analysis = analyzeTestResults(this.testResults);
+    const duration = Date.now() - this.startTime;
+
+    const testModulesErrors = testModules.flatMap((t) => t.errors());
+    const unhandledErrorCount = unhandledErrors.length + testModulesErrors.length;
+
+    // Fire and forget — same pattern as the existing test-run telemetry
+    telemetry(
+      'ai-setup-self-healing-scoring',
+      {
+        agent: this.agent,
+        analysis,
+        unhandledErrorCount,
+        duration,
+        watch: this.ctx.config.watch,
+      },
+      { configDir: this.configDir, stripMetadata: true }
+    );
+
+    // Reset for next run (watch mode)
+    this.testResults = [];
+  }
+}

code/addons/vitest/src/vitest-plugin/index.ts

@@ -20,8 +20,14 @@ import {
 } from 'storybook/internal/core-server';
 import { componentTransform, readConfig, vitestTransform } from 'storybook/internal/csf-tools';
 import { MainFileMissingError } from 'storybook/internal/server-errors';
-import { setTelemetryEnabled, telemetry } from 'storybook/internal/telemetry';
-import { oneWayHash } from 'storybook/internal/telemetry';
+import {
+  detectAgent,
+  isTelemetryModuleEnabled,
+  isWithinInitialSession,
+  oneWayHash,
+  telemetry,
+  setTelemetryEnabled,
+} from 'storybook/internal/telemetry';
 import type { Presets } from 'storybook/internal/types';
 
 import { match } from 'micromatch';
@@ -36,6 +42,7 @@ import type { PluginOption } from 'vite';
 import { withoutVitePlugins } from '../../../../builders/builder-vite/src/utils/without-vite-plugins.ts';
 import type { InternalOptions, UserOptions } from './types.ts';
 import { requiresProjectAnnotations } from './utils.ts';
+import { AgentTelemetryReporter } from './agent-telemetry-reporter.ts';
 
 const WORKING_DIR = process.cwd();
 
@@ -241,6 +248,8 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
     plugins.push(mdxStubPlugin);
   }
 
+  let withinAgenticSetupSession = false;
+
   const storybookTestPlugin: Plugin = {
     name: 'vite-plugin-storybook-test',
     async transformIndexHtml(html) {
@@ -385,6 +394,15 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
                   globals.ghostStories = {
                     enabled: true,
                   };
+                  globals.renderAnalysis = {
+                    enabled: true,
+                  };
+                }
+
+                if (withinAgenticSetupSession) {
+                  globals.renderAnalysis = {
+                    enabled: true,
+                  };
                 }
 
                 return globals;
@@ -441,7 +459,7 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
       // return the new config, it will be deep-merged by vite
       return config;
     },
-    configureVitest(context) {
+    async configureVitest(context) {
       context.vitest.config.coverage.exclude.push('storybook-static');
 
       // NOTE: we start telemetry immediately but do not wait on it. Typically it should complete
@@ -455,6 +473,21 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
         },
         { configDir: finalOptions.configDir }
       );
+
+      if (isTelemetryModuleEnabled()) {
+        // When an agent is running vitest via CLI, inject a reporter that sends
+        // detailed test result telemetry (pass/fail, error analysis, empty renders)
+        const agent = detectAgent();
+        withinAgenticSetupSession = !!agent && (await isWithinInitialSession('ai-setup'));
+        if (agent && withinAgenticSetupSession) {
+          context.vitest.config.reporters.push(
+            new AgentTelemetryReporter({
+              configDir: finalOptions.configDir,
+              agent,
+            })
+          );
+        }
+      }
     },
     async configureServer(server) {
       if (staticDirs) {

code/core/src/core-server/index.ts

@@ -37,5 +37,9 @@ export {
 } from './stores/test-provider.ts';
 
 export { getComponentCandidates } from './utils/ghost-stories/get-candidates.ts';
-export { runGhostStories } from './utils/ghost-stories/run-story-tests.ts';
+export { runStoryTests } from './utils/ghost-stories/run-story-tests.ts';
 export { getServerPort } from './utils/server-address.ts';
+
+export { analyzeTestResults } from '../shared/utils/analyze-test-results.ts';
+export type { StoryTestResult } from '../shared/utils/test-result-types.ts';
+export { toStoryTestResult } from '../shared/utils/to-story-test-result.ts';