Core: Agentic observability for vitest and ghost stories

code/addons/a11y/src/preview.tsx

@@ -20,10 +20,10 @@ export const afterEach: AfterEach<any> = async ({
 }) => {
   const a11yParameter: A11yParameters | undefined = parameters.a11y;
   const a11yGlobals = globals.a11y;
-  const isGhostStories = !!globals.ghostStories;
+  const isInternalRenderAnalysis = !!globals.renderAnalysis;
 
   const shouldRunEnvironmentIndependent =
-    !isGhostStories &&
+    !isInternalRenderAnalysis &&
     a11yParameter?.disable !== true &&
     a11yParameter?.test !== 'off' &&
     a11yGlobals?.manual !== true;

code/addons/vitest/src/vitest-plugin/agent-telemetry-reporter.test.ts

@@ -0,0 +1,252 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { AgentTelemetryReporter, extractErrorMessage } from './agent-telemetry-reporter.ts';
+
+vi.mock('storybook/internal/telemetry', () => ({
+  telemetry: vi.fn(),
+  isExampleStoryId: vi.fn(
+    (id: string) =>
+      id.startsWith('example-button--') ||
+      id.startsWith('example-header--') ||
+      id.startsWith('example-page--')
+  ),
+}));
+
+const { telemetry } = await import('storybook/internal/telemetry');
+
+function createMockTestCase({
+  storyId,
+  status,
+  reports = [],
+  errors = [],
+}: {
+  storyId?: string;
+  status: 'passed' | 'failed' | 'pending';
+  reports?: Array<{ type: string; result?: Record<string, unknown> }>;
+  errors?: Array<{ message: string; stack?: string }>;
+}) {
+  return {
+    meta: () => ({ storyId, reports }),
+    result: () => ({
+      state: status,
+      errors: status === 'failed' ? errors : [],
+    }),
+  };
+}
+
+function createMockTestModules(testCounts: { passed: number; failed: number }) {
+  const tests: Array<{ result: () => { state: string } }> = [];
+  for (let i = 0; i < testCounts.passed; i++) {
+    tests.push({ result: () => ({ state: 'passed' }) });
+  }
+  for (let i = 0; i < testCounts.failed; i++) {
+    tests.push({ result: () => ({ state: 'failed' }) });
+  }
+  return [
+    {
+      children: {
+        allTests: function* (filter?: string) {
+          for (const t of tests) {
+            if (!filter || t.result().state === filter) {
+              yield t;
+            }
+          }
+        },
+      },
+      errors: () => [],
+    },
+  ];
+}
+
+describe('extractErrorMessage', () => {
+  it('returns the first line of a plain message', () => {
+    expect(extractErrorMessage('TypeError: foo is not a function\n    at bar', undefined)).toBe(
+      'TypeError: foo is not a function'
+    );
+  });
+
+  it('strips the Storybook debug banner and returns the actual message', () => {
+    const message =
+      '\n\x1B[34mClick to debug the error directly in Storybook: http://localhost:6006/?path=/story/button--primary\x1B[39m\n\nmissing theme context provider';
+    expect(extractErrorMessage(message, undefined)).toBe('missing theme context provider');
+  });
+
+  it('falls back to the first line of the stack when message is empty', () => {
+    expect(extractErrorMessage('', 'Error: something broke\n    at foo')).toBe(
+      'Error: something broke'
+    );
+  });
+
+  it('falls back to the first line of the stack when message is undefined', () => {
+    expect(extractErrorMessage(undefined, 'Error: something broke\n    at foo')).toBe(
+      'Error: something broke'
+    );
+  });
+
+  it('returns "unknown error" when both message and stack are empty', () => {
+    expect(extractErrorMessage('', undefined)).toBe('unknown error');
+  });
+
+  it('returns "unknown error" when both message and stack are undefined', () => {
+    expect(extractErrorMessage(undefined, undefined)).toBe('unknown error');
+  });
+
+  it('handles a banner where the actual message itself is multi-line', () => {
+    const message =
+      '\n\x1B[34mClick to debug the error directly in Storybook: http://localhost:6006/?path=/story/button--primary\x1B[39m\n\nfirst error line\nsecond line';
+    expect(extractErrorMessage(message, undefined)).toBe('first error line');
+  });
+
+  it('falls back to stack when message starts with a newline but has no banner', () => {
+    expect(extractErrorMessage('\nsome error', 'Error: fallback\n    at foo')).toBe(
+      'Error: fallback'
+    );
+  });
+});
+
+describe('AgentTelemetryReporter', () => {
+  let reporter: AgentTelemetryReporter;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    reporter = new AgentTelemetryReporter({
+      configDir: '.storybook',
+      agent: { name: 'claude' },
+    });
+  });
+
+  describe('onTestCaseResult', () => {
+    it('should collect story test results', () => {
+      const testCase = createMockTestCase({
+        storyId: 'my-story--primary',
+        status: 'passed',
+      });
+      reporter.onTestCaseResult(testCase as any);
+    });
+
+    it('should skip tests without storyId', () => {
+      const testCase = createMockTestCase({
+        storyId: undefined,
+        status: 'passed',
+      });
+      reporter.onTestCaseResult(testCase as any);
+    });
+
+    it('should skip example story IDs', () => {
+      const testCase = createMockTestCase({
+        storyId: 'example-button--primary',
+        status: 'passed',
+      });
+      reporter.onTestCaseResult(testCase as any);
+    });
+  });
+
+  describe('onTestRunEnd', () => {
+    it('should send telemetry with analysis of collected results', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      reporter.onTestCaseResult(createMockTestCase({ storyId: 's1', status: 'passed' }) as any);
+      reporter.onTestCaseResult(
+        createMockTestCase({
+          storyId: 's2',
+          status: 'failed',
+          errors: [{ message: 'Error: Module not found: foo' }],
+        }) as any
+      );
+      reporter.onTestCaseResult(
+        createMockTestCase({
+          storyId: 's3',
+          status: 'passed',
+          reports: [{ type: 'render-analysis', result: { emptyRender: true } }],
+        }) as any
+      );
+
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 2, failed: 1 }) as any, []);
+
+      expect(telemetry).toHaveBeenCalledWith(
+        'agent-test-run',
+        expect.objectContaining({
+          agent: { name: 'claude' },
+          analysis: expect.objectContaining({
+            total: 3,
+            passed: 2,
+            passedButEmptyRender: 1,
+            successRate: 0.67,
+            successRateWithoutEmptyRender: 0.33,
+            uniqueErrorCount: 1,
+          }),
+          unhandledErrorCount: 0,
+          watch: false,
+        }),
+        { configDir: '.storybook', stripMetadata: true }
+      );
+    });
+
+    it('should filter out example stories from analysis', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      reporter.onTestCaseResult(
+        createMockTestCase({ storyId: 'my-story--primary', status: 'passed' }) as any
+      );
+      reporter.onTestCaseResult(
+        createMockTestCase({ storyId: 'example-button--primary', status: 'passed' }) as any
+      );
+
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 2, failed: 0 }) as any, []);
+
+      expect(telemetry).toHaveBeenCalledWith(
+        'agent-test-run',
+        expect.objectContaining({
+          analysis: expect.objectContaining({
+            total: 1,
+            passed: 1,
+          }),
+        }),
+        expect.anything()
+      );
+    });
+
+    it('should count unhandled errors', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      await reporter.onTestRunEnd(
+        createMockTestModules({ passed: 0, failed: 0 }) as any,
+        [{ message: 'unhandled' }, { message: 'another' }] as any
+      );
+
+      expect(telemetry).toHaveBeenCalledWith(
+        'agent-test-run',
+        expect.objectContaining({
+          unhandledErrorCount: 2,
+        }),
+        expect.anything()
+      );
+    });
+
+    it('should reset collected results after each run', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      reporter.onTestCaseResult(createMockTestCase({ storyId: 's1', status: 'passed' }) as any);
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 1, failed: 0 }) as any, []);
+
+      reporter.onTestCaseResult(
+        createMockTestCase({
+          storyId: 's2',
+          status: 'failed',
+          errors: [{ message: 'err' }],
+        }) as any
+      );
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 0, failed: 1 }) as any, []);
+
+      const secondCall = vi.mocked(telemetry).mock.calls[1];
+      expect(secondCall[1]).toEqual(
+        expect.objectContaining({
+          analysis: expect.objectContaining({
+            total: 1,
+            passed: 0,
+          }),
+        })
+      );
+    });
+  });
+});

code/addons/vitest/src/vitest-plugin/agent-telemetry-reporter.ts

@@ -0,0 +1,131 @@
+import type { SerializedError } from 'vitest';
+import type { TestCase, TestModule, Vitest } from 'vitest/node';
+import type { Reporter } from 'vitest/reporters';
+
+import type { TaskMeta } from '@vitest/runner';
+import type { Report } from 'storybook/preview-api';
+import { analyzeTestResults } from 'storybook/internal/core-server';
+import type { StoryTestResult } from 'storybook/internal/core-server';
+import { isExampleStoryId, telemetry } from 'storybook/internal/telemetry';
+import type { AgentInfo } from 'storybook/internal/telemetry';
+
+/**
+ * Extracts a clean single-line error message from a Vitest error.
+ *
+ * Important to handle scenarios e.g. stripping out a "Click to debug" banner and extracting the actual error message.
+ */
+export function extractErrorMessage(
+  message: string | undefined,
+  stack: string | undefined
+): string {
+  let rawMessage = message ?? '';
+
+  // Strip the Storybook debug banner if present
+  if (rawMessage.startsWith('\n\x1B[34m')) {
+    const bannerEnd = rawMessage.indexOf('\n\n');
+    if (bannerEnd !== -1) {
+      rawMessage = rawMessage.slice(bannerEnd + 2);
+    }
+  }
+
+  return rawMessage.split('\n')[0] || stack?.split('\n')[0] || 'unknown error';
+}
+
+interface AgentTelemetryReporterOptions {
+  configDir: string;
+  agent: AgentInfo;
+}
+
+export class AgentTelemetryReporter implements Reporter {
+  private ctx!: Vitest;
+
+  private testResults: StoryTestResult[] = [];
+
+  private startTime = Date.now();
+
+  private configDir: string;
+
+  private agent: AgentInfo;
+
+  constructor(options: AgentTelemetryReporterOptions) {
+    this.configDir = options.configDir;
+    this.agent = options.agent;
+  }
+
+  onInit(ctx: Vitest) {
+    this.ctx = ctx;
+    this.startTime = Date.now();
+  }
+
+  onTestCaseResult(testCase: TestCase) {
+    const { storyId, reports } = testCase.meta() as TaskMeta &
+      Partial<{ storyId: string; reports: Report[] }>;
+
+    // Silently skip non-story tests
+    if (!storyId) {
+      return;
+    }
+
+    // Filter out scaffold example stories
+    if (isExampleStoryId(storyId)) {
+      return;
+    }
+
+    const testResult = testCase.result();
+    const status =
+      testResult.state === 'passed' ? 'PASS' : testResult.state === 'failed' ? 'FAIL' : 'PENDING';
+
+    // Detect empty render from reports
+    const emptyRender =
+      status === 'PASS' &&
+      reports?.some(
+        (report) =>
+          report.type === 'render-analysis' && (report.result as any)?.emptyRender === true
+      );
+
+    // Extract error message (first line) and stack
+    let error: string | undefined;
+    let stack: string | undefined;
+    if (testResult.errors && testResult.errors.length > 0) {
+      const firstError = testResult.errors[0];
+      error = extractErrorMessage(firstError.message, firstError.stack);
+      stack = firstError.stack;
+    }
+
+    this.testResults.push({
+      storyId,
+      status,
+      error,
+      stack,
+      emptyRender: emptyRender || undefined,
+    });
+  }
+
+  async onTestRunEnd(
+    testModules: readonly TestModule[],
+    unhandledErrors: readonly SerializedError[]
+  ) {
+    const analysis = analyzeTestResults(this.testResults);
+    const duration = Date.now() - this.startTime;
+
+    const testModulesErrors = testModules.flatMap((t) => t.errors());
+    const unhandledErrorCount = unhandledErrors.length + testModulesErrors.length;
+
+    // Fire and forget — same pattern as the existing test-run telemetry
+    telemetry(
+      'agent-test-run',
+      {
+        agent: this.agent,
+        analysis,
+        unhandledErrorCount,
+        duration,
+        watch: this.ctx.config.watch,
+      },
+      { configDir: this.configDir, stripMetadata: true }
+    );
+
+    // Reset for next run (watch mode)
+    this.testResults = [];
+    this.startTime = Date.now();
+  }
+}